diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java index 0fb640234db..0f4bdaee7af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java @@ -30,8 +30,6 @@ public interface ApplicationApi { Response getApplications(String state); - Response getApplication(String appName); - Response deleteApplication(String appName); Response updateApplication(String appName, Application updateAppData); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java index 5a4726ea385..b4f6a2ee384 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java @@ -17,23 +17,26 @@ package org.apache.hadoop.yarn.services.api.impl; -import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*; -import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.lang.reflect.UndeclaredThrowableException; -import java.security.PrivilegedExceptionAction; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.regex.Pattern; +import com.google.inject.Singleton; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.ApplicationState; +import org.apache.slider.api.resource.ApplicationStatus; +import org.apache.slider.api.resource.Component; +import org.apache.slider.util.ServiceApiUtil; +import org.apache.slider.client.SliderClient; +import org.apache.slider.common.params.ActionFreezeArgs; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.common.tools.SliderVersionInfo; +import org.apache.slider.core.buildutils.BuildHelper; +import org.apache.slider.core.exceptions.SliderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; @@ -43,77 +46,26 @@ import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; -import org.apache.commons.lang.SerializationUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.services.api.ApplicationApi; -import org.apache.slider.api.resource.Application; -import org.apache.slider.api.resource.ApplicationState; -import org.apache.slider.api.resource.ApplicationStatus; -import org.apache.slider.api.resource.Artifact; -import org.apache.slider.api.resource.Component; -import org.apache.slider.api.resource.ConfigFile; -import org.apache.slider.api.resource.Configuration; -import org.apache.slider.api.resource.Container; -import org.apache.slider.api.resource.ContainerState; -import org.apache.slider.api.resource.Resource; -import org.apache.slider.api.OptionKeys; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.api.StateValues; -import org.apache.slider.client.SliderClient; -import org.apache.slider.common.SliderExitCodes; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.params.ActionCreateArgs; -import org.apache.slider.common.params.ActionFlexArgs; -import org.apache.slider.common.params.ActionFreezeArgs; -import org.apache.slider.common.params.ActionListArgs; -import org.apache.slider.common.params.ActionRegistryArgs; -import org.apache.slider.common.params.ActionThawArgs; -import org.apache.slider.common.params.ActionUpdateArgs; -import org.apache.slider.common.params.ComponentArgsDelegate; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.common.tools.SliderVersionInfo; -import org.apache.slider.core.buildutils.BuildHelper; -import org.apache.slider.core.exceptions.BadClusterStateException; -import org.apache.slider.core.exceptions.NotFoundException; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.exceptions.UnknownApplicationInstanceException; -import org.apache.slider.core.registry.docstore.ConfigFormat; -import org.apache.slider.providers.docker.DockerKeys; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.annotations.VisibleForTesting; -import com.google.gson.JsonElement; -import com.google.gson.JsonNull; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.inject.Singleton; +import static org.apache.slider.util.RestApiConstants.*; @Singleton @Path(APPLICATIONS_API_RESOURCE_PATH) @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) -public class ApplicationApiService implements ApplicationApi { - private static final Logger logger = LoggerFactory - .getLogger(ApplicationApiService.class); - private static org.apache.hadoop.conf.Configuration SLIDER_CONFIG; - private static UserGroupInformation SLIDER_USER; +public class ApplicationApiService { + private static final Logger logger = + LoggerFactory.getLogger(ApplicationApiService.class); + private static org.apache.hadoop.conf.Configuration SLIDER_CONFIG = + new YarnConfiguration(); private static SliderClient SLIDER_CLIENT; private static Response SLIDER_VERSION; - private static final JsonParser JSON_PARSER = new JsonParser(); - private static final JsonObject EMPTY_JSON_OBJECT = new JsonObject(); - private static final ActionListArgs ACTION_LIST_ARGS = new ActionListArgs(); private static final ActionFreezeArgs ACTION_FREEZE_ARGS = new ActionFreezeArgs(); static { @@ -122,8 +74,6 @@ public class ApplicationApiService implements ApplicationApi { // initialize all the common resources - order is important protected static void init() { - SLIDER_CONFIG = getSliderClientConfiguration(); - SLIDER_USER = getSliderUser(); SLIDER_CLIENT = createSliderClient(); SLIDER_VERSION = initSliderVersion(); } @@ -131,8 +81,7 @@ public class ApplicationApiService implements ApplicationApi { @GET @Path("/versions/slider-version") @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON }) - public Response getSliderVersion() { + @Produces({ MediaType.APPLICATION_JSON }) public Response getSliderVersion() { logger.info("GET: getSliderVersion"); return SLIDER_VERSION; } @@ -148,580 +97,45 @@ public class ApplicationApiService implements ApplicationApi { + "\", \"hadoop_version\": \"" + hadoopVersion + "\"}").build(); } - @POST - @Consumes({ MediaType.APPLICATION_JSON }) + @POST @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response createApplication(Application application) { - logger.info("POST: createApplication for app = {}", application); + logger.info("POST: createApplication = {}", application); ApplicationStatus applicationStatus = new ApplicationStatus(); - - Map compNameArtifactIdMap = new HashMap<>(); - // post payload validation try { - validateApplicationPostPayload(application, compNameArtifactIdMap); + ApplicationId applicationId = SLIDER_CLIENT.actionCreate(application); + logger.info("Successfully created application " + application.getName() + + " applicationId = " + applicationId); + applicationStatus.setState(ApplicationState.ACCEPTED); + applicationStatus.setUri( + CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH + "/" + application + .getName()); + return Response.status(Status.CREATED).entity(applicationStatus).build(); } catch (IllegalArgumentException e) { applicationStatus.setDiagnostics(e.getMessage()); return Response.status(Status.BAD_REQUEST).entity(applicationStatus) .build(); - } - String applicationId = null; - try { - applicationId = createSliderApp(application, compNameArtifactIdMap); - applicationStatus.setState(ApplicationState.ACCEPTED); - } catch (SliderException se) { - logger.error("Create application failed", se); - if (se.getExitCode() == SliderExitCodes.EXIT_APPLICATION_IN_USE) { - applicationStatus.setDiagnostics(ERROR_APPLICATION_IN_USE); - return Response.status(Status.BAD_REQUEST).entity(applicationStatus) - .build(); - } else if (se.getExitCode() == SliderExitCodes.EXIT_INSTANCE_EXISTS) { - applicationStatus.setDiagnostics(ERROR_APPLICATION_INSTANCE_EXISTS); - return Response.status(Status.BAD_REQUEST).entity(applicationStatus) - .build(); - } else { - applicationStatus.setDiagnostics(se.getMessage()); - } } catch (Exception e) { - logger.error("Create application failed", e); - applicationStatus.setDiagnostics(e.getMessage()); - } - - if (StringUtils.isNotEmpty(applicationId)) { - applicationStatus.setUri(CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH - + "/" + application.getName()); - // 202 = ACCEPTED - return Response.status(HTTP_STATUS_CODE_ACCEPTED) - .entity(applicationStatus).build(); - } else { + String message = "Failed to create application " + application.getName(); + logger.error(message, e); + applicationStatus.setDiagnostics(message + ": " + e.getMessage()); return Response.status(Status.INTERNAL_SERVER_ERROR) .entity(applicationStatus).build(); } } - @VisibleForTesting - protected void validateApplicationPostPayload(Application application, - Map compNameArtifactIdMap) { - if (StringUtils.isEmpty(application.getName())) { - throw new IllegalArgumentException(ERROR_APPLICATION_NAME_INVALID); - } - if (!SliderUtils.isClusternameValid(application.getName())) { - throw new IllegalArgumentException(ERROR_APPLICATION_NAME_INVALID_FORMAT); - } - - // If the application has no components do top-level checks - if (application.getComponents() == null - || application.getComponents().size() == 0) { - // artifact - if (application.getArtifact() == null) { - throw new IllegalArgumentException(ERROR_ARTIFACT_INVALID); - } - if (StringUtils.isEmpty(application.getArtifact().getId())) { - throw new IllegalArgumentException(ERROR_ARTIFACT_ID_INVALID); - } - - // If artifact is of type APPLICATION, add a slider specific property - if (application.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) { - if (application.getConfiguration() == null) { - application.setConfiguration(new Configuration()); - } - addPropertyToConfiguration(application.getConfiguration(), - SliderKeys.COMPONENT_TYPE_KEY, - SliderKeys.COMPONENT_TYPE_EXTERNAL_APP); - } - // resource - validateApplicationResource(application.getResource(), null, application - .getArtifact().getType()); - - // container size - if (application.getNumberOfContainers() == null) { - throw new IllegalArgumentException(ERROR_CONTAINERS_COUNT_INVALID); - } - - // Since it is a simple app with no components, create a default component - application.setComponents(getDefaultComponentAsList(application)); - } else { - // If the application has components, then run checks for each component. - // Let global values take effect if component level values are not - // provided. - Artifact globalArtifact = application.getArtifact(); - Resource globalResource = application.getResource(); - Long globalNumberOfContainers = application.getNumberOfContainers(); - for (Component comp : application.getComponents()) { - // artifact - if (comp.getArtifact() == null) { - comp.setArtifact(globalArtifact); - } - // If still null raise validation exception - if (comp.getArtifact() == null) { - throw new IllegalArgumentException(String.format( - ERROR_ARTIFACT_FOR_COMP_INVALID, comp.getName())); - } - if (StringUtils.isEmpty(comp.getArtifact().getId())) { - throw new IllegalArgumentException(String.format( - ERROR_ARTIFACT_ID_FOR_COMP_INVALID, comp.getName())); - } - - // If artifact is of type APPLICATION, add a slider specific property - if (comp.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) { - if (comp.getConfiguration() == null) { - comp.setConfiguration(new Configuration()); - } - addPropertyToConfiguration(comp.getConfiguration(), - SliderKeys.COMPONENT_TYPE_KEY, - SliderKeys.COMPONENT_TYPE_EXTERNAL_APP); - compNameArtifactIdMap.put(comp.getName(), comp.getArtifact().getId()); - comp.setName(comp.getArtifact().getId()); - } - - // resource - if (comp.getResource() == null) { - comp.setResource(globalResource); - } - validateApplicationResource(comp.getResource(), comp, comp - .getArtifact().getType()); - - // container count - if (comp.getNumberOfContainers() == null) { - comp.setNumberOfContainers(globalNumberOfContainers); - } - if (comp.getNumberOfContainers() == null) { - throw new IllegalArgumentException(String.format( - ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID, comp.getName())); - } - } - } - - // Application lifetime if not specified, is set to unlimited lifetime - if (application.getLifetime() == null) { - application.setLifetime(DEFAULT_UNLIMITED_LIFETIME); - } - } - - private void validateApplicationResource(Resource resource, Component comp, - Artifact.TypeEnum artifactType) { - // Only apps/components of type APPLICATION can skip resource requirement - if (resource == null && artifactType == Artifact.TypeEnum.APPLICATION) { - return; - } - if (resource == null) { - throw new IllegalArgumentException(comp == null ? ERROR_RESOURCE_INVALID - : String.format(ERROR_RESOURCE_FOR_COMP_INVALID, comp.getName())); - } - // One and only one of profile OR cpus & memory can be specified. Specifying - // both raises validation error. - if (StringUtils.isNotEmpty(resource.getProfile()) - && (resource.getCpus() != null - || StringUtils.isNotEmpty(resource.getMemory()))) { - throw new IllegalArgumentException( - comp == null ? ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED - : String.format( - ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED, - comp.getName())); - } - // Currently resource profile is not supported yet, so we will raise - // validation error if only resource profile is specified - if (StringUtils.isNotEmpty(resource.getProfile())) { - throw new IllegalArgumentException( - ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET); - } - - String memory = resource.getMemory(); - Integer cpus = resource.getCpus(); - if (StringUtils.isEmpty(memory)) { - throw new IllegalArgumentException( - comp == null ? ERROR_RESOURCE_MEMORY_INVALID : String.format( - ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID, comp.getName())); - } - if (cpus == null) { - throw new IllegalArgumentException( - comp == null ? ERROR_RESOURCE_CPUS_INVALID : String.format( - ERROR_RESOURCE_CPUS_FOR_COMP_INVALID, comp.getName())); - } - if (cpus <= 0) { - throw new IllegalArgumentException( - comp == null ? ERROR_RESOURCE_CPUS_INVALID_RANGE : String.format( - ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE, comp.getName())); - } - } - - private String createSliderApp(Application application, - Map compNameArtifactIdMap) throws IOException, - YarnException, InterruptedException { - final String appName = application.getName(); - final String queueName = application.getQueue(); - - final ActionCreateArgs createArgs = new ActionCreateArgs(); - addAppConfOptions(createArgs, application, compNameArtifactIdMap); - addResourceOptions(createArgs, application); - - createArgs.provider = DockerKeys.PROVIDER_DOCKER; - - if (queueName != null && queueName.trim().length() > 0) { - createArgs.queue = queueName.trim(); - } - createArgs.lifetime = application.getLifetime(); - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public String run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - sliderClient.actionCreate(appName, createArgs); - ApplicationId applicationId = sliderClient.applicationId; - if (applicationId != null) { - return applicationId.toString(); - // return getApplicationIdString(applicationId); - } - return null; - } - }); - } - - private void addAppConfOptions(ActionCreateArgs createArgs, - Application application, Map compNameArtifactIdMap) throws IOException { - List appCompOptionTriples = createArgs.optionsDelegate.compOptTriples; // TODO: optionTuples instead of compOptTriples - logger.info("Initial appCompOptionTriples = {}", - Arrays.toString(appCompOptionTriples.toArray())); - List appOptions = createArgs.optionsDelegate.optionTuples; - logger.info("Initial appOptions = {}", - Arrays.toString(appOptions.toArray())); - // TODO: Set Slider-AM memory and vcores here - // appCompOptionTriples.addAll(Arrays.asList(SLIDER_APPMASTER_COMPONENT_NAME, - // "", "")); - - // Global configuration - for override purpose - // TODO: add it to yaml - Configuration globalConfig = null; - // Configuration globalConfig = (Configuration) SerializationUtils - // .clone(application.getConfiguration()); - - // TODO: Add the below into globalConfig - // if (application.getConfigurations() != null) { - // for (Entry entry : application.getConfigurations() - // .entrySet()) { - // globalConf.addProperty(entry.getKey(), entry.getValue()); - // } - // } - - Set uniqueGlobalPropertyCache = new HashSet<>(); - if (application.getConfiguration() != null) { - if (application.getConfiguration().getProperties() != null) { - for (Map.Entry propEntry : application - .getConfiguration().getProperties().entrySet()) { - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - propEntry.getKey(), propEntry.getValue()); - } - } - List configFiles = application.getConfiguration().getFiles(); - if (configFiles != null && !configFiles.isEmpty()) { - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - SliderKeys.AM_CONFIG_GENERATION, "true"); - for (ConfigFile configFile : configFiles) { - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - OptionKeys.CONF_FILE_PREFIX + configFile.getSrcFile() + - OptionKeys.NAME_SUFFIX, configFile.getDestFile()); - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - OptionKeys.CONF_FILE_PREFIX + configFile.getSrcFile() + - OptionKeys.TYPE_SUFFIX, configFile.getType().toString()); - } - } - } - if (application.getComponents() != null) { - - Map appQuicklinks = application.getQuicklinks(); - if (appQuicklinks != null) { - for (Map.Entry quicklink : appQuicklinks.entrySet()) { - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - OptionKeys.EXPORT_PREFIX + quicklink.getKey(), - quicklink.getValue()); - } - } - - Map placeholders = new HashMap<>(); - placeholders.put(PLACEHOLDER_APP_NAME, application.getName()); - for (Component comp : application.getComponents()) { - placeholders.put(PLACEHOLDER_APP_COMPONENT_NAME, comp.getName()); - if (comp.getArtifact().getType() == Artifact.TypeEnum.DOCKER) { - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - DockerKeys.DOCKER_IMAGE, comp.getArtifact().getId() == null ? - application.getArtifact().getId() : comp.getArtifact().getId())); - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - DockerKeys.DOCKER_START_COMMAND, comp.getLaunchCommand() == null ? - replacePlaceholders(application.getLaunchCommand(), placeholders) - : replacePlaceholders(comp.getLaunchCommand(), placeholders))); - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - DockerKeys.DOCKER_NETWORK, DockerKeys.DEFAULT_DOCKER_NETWORK)); - if (comp.getRunPrivilegedContainer() != null) { - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - DockerKeys.DOCKER_USE_PRIVILEGED, - comp.getRunPrivilegedContainer().toString())); - } - } - - if (comp.getConfiguration() != null) { - List configFiles = comp.getConfiguration().getFiles(); - if (configFiles != null && !configFiles.isEmpty()) { - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - SliderKeys.AM_CONFIG_GENERATION, "true")); - for (ConfigFile configFile : configFiles) { - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - OptionKeys.CONF_FILE_PREFIX + configFile.getSrcFile() + - OptionKeys.NAME_SUFFIX, configFile.getDestFile())); - appCompOptionTriples.addAll(Arrays.asList(comp.getName(), - OptionKeys.CONF_FILE_PREFIX + configFile.getSrcFile() + - OptionKeys.TYPE_SUFFIX, configFile.getType().toString())); - } - } - } - - if (Boolean.TRUE.equals(comp.getUniqueComponentSupport())) { - for (int i = 1; i <= comp.getNumberOfContainers(); i++) { - placeholders.put(PLACEHOLDER_COMPONENT_ID, Integer.toString(i)); - appCompOptionTriples.addAll(createAppConfigComponent( - comp.getName() + i, comp, comp.getName() + i, globalConfig, - placeholders, compNameArtifactIdMap)); - } - } else { - appCompOptionTriples.addAll(createAppConfigComponent(comp.getName(), - comp, comp.getName(), globalConfig, null, compNameArtifactIdMap)); - } - } - } - - logger.info("Updated appCompOptionTriples = {}", - Arrays.toString(appCompOptionTriples.toArray())); - logger.info("Updated appOptions = {}", - Arrays.toString(appOptions.toArray())); - } - - private void addOptionsIfNotPresent(List options, - Set uniqueGlobalPropertyCache, String key, String value) { - if (uniqueGlobalPropertyCache == null) { - options.addAll(Arrays.asList(key, value)); - } else if (!uniqueGlobalPropertyCache.contains(key)) { - options.addAll(Arrays.asList(key, value)); - uniqueGlobalPropertyCache.add(key); - } - } - - private void addPropertyToConfiguration(Configuration conf, String key, - String value) { - if (conf == null) { - return; - } - if (conf.getProperties() == null) { - conf.setProperties(new HashMap()); - } - conf.getProperties().put(key, value); - } - - private List createAppConfigComponent(String compName, - Component component, String configPrefix, Configuration globalConf, - Map placeholders, - Map compNameArtifactIdMap) { - List appConfOptTriples = new ArrayList<>(); - - if (component.getConfiguration() != null - && component.getConfiguration().getProperties() != null) { - for (Map.Entry propEntry : component.getConfiguration() - .getProperties().entrySet()) { - appConfOptTriples.addAll(Arrays.asList(compName, propEntry.getKey(), - replacePlaceholders(propEntry.getValue(), placeholders))); - } - } - - // If artifact is of type APPLICATION, then in the POST JSON there will - // be no component definition for that artifact. Hence it's corresponding id - // field is added. Every external APPLICATION has a unique id field. - List convertedDeps = new ArrayList<>(); - for (String dep : component.getDependencies()) { - if (compNameArtifactIdMap.containsKey(dep)) { - convertedDeps.add(compNameArtifactIdMap.get(dep)); - } else { - convertedDeps.add(dep); - } - } - // If the DNS dependency property is set to true for a component, it means - // that it is ensured that DNS entry has been added for all the containers - // of this component, before moving on to the next component in the DAG. - if (hasPropertyWithValue(component, PROPERTY_DNS_DEPENDENCY, "true")) { - if (component.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) { - convertedDeps.add(component.getArtifact().getId()); - } else { - convertedDeps.add(compName); - } - } - if (convertedDeps.size() > 0) { - appConfOptTriples.addAll(Arrays.asList(compName, "requires", - StringUtils.join(convertedDeps, ","))); - } - return appConfOptTriples; - } - - private String replacePlaceholders(String value, - Map placeholders) { - if (StringUtils.isEmpty(value) || placeholders == null) { - return value; - } - for (Map.Entry placeholder : placeholders.entrySet()) { - value = value.replaceAll(Pattern.quote(placeholder.getKey()), - placeholder.getValue()); - } - return value; - } - - private List createAppConfigGlobal(Component component, - Configuration globalConf, Set uniqueGlobalPropertyCache) { - List appOptions = new ArrayList<>(); - if (component.getConfiguration() != null - && component.getConfiguration().getProperties() != null) { - for (Map.Entry propEntry : component.getConfiguration() - .getProperties().entrySet()) { - addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache, - propEntry.getKey(), propEntry.getValue()); - } - } - return appOptions; - } - - private void addResourceOptions(ActionCreateArgs createArgs, - Application application) throws IOException { - List resCompOptionTriples = createArgs.optionsDelegate.resCompOptTriples; - logger.info("Initial resCompOptTriples = {}", - Arrays.toString(resCompOptionTriples.toArray())); - // TODO: Add any Slider AM resource specific props here like jvm.heapsize - // resCompOptionTriples.addAll(Arrays.asList(SLIDER_APPMASTER_COMPONENT_NAME, - // "", "")); - - // Global resource - for override purpose - Resource globalResource = (Resource) SerializationUtils.clone(application - .getResource()); - // Priority seeded with 1, expecting every new component will increase it by - // 1 making it ready for the next component to use. - if (application.getComponents() != null) { - int priority = 1; - for (Component comp : application.getComponents()) { - if (hasPropertyWithValue(comp, SliderKeys.COMPONENT_TYPE_KEY, - SliderKeys.COMPONENT_TYPE_EXTERNAL_APP)) { - continue; - } - if (Boolean.TRUE.equals(comp.getUniqueComponentSupport())) { - for (int i = 1; i <= comp.getNumberOfContainers(); i++) { - resCompOptionTriples.addAll(createResourcesComponent(comp.getName() - + i, comp, priority, 1, globalResource)); - priority++; - } - } else { - resCompOptionTriples.addAll(createResourcesComponent(comp.getName(), - comp, priority, comp.getNumberOfContainers(), globalResource)); - priority++; - } - } - } - - logger.info("Updated resCompOptTriples = {}", - Arrays.toString(resCompOptionTriples.toArray())); - } - - private boolean hasPropertyWithValue(Component comp, String key, String value) { - if (comp == null || key == null) { - return false; - } - if (comp.getConfiguration() == null - || comp.getConfiguration().getProperties() == null) { - return false; - } - Map props = comp.getConfiguration().getProperties(); - if (props.containsKey(key)) { - if (value == null) { - return props.get(key) == null; - } else { - if (value.equals(props.get(key))) { - return true; - } - } - } - return false; - } - - private List createResourcesComponent(String compName, - Component component, int priority, long numInstances, - Resource globalResource) { - String memory = component.getResource() == null ? globalResource - .getMemory() : component.getResource().getMemory(); - Integer cpus = component.getResource() == null ? globalResource.getCpus() - : component.getResource().getCpus(); - - List resCompOptTriples = new ArrayList(); - resCompOptTriples.addAll(Arrays.asList(compName, - ResourceKeys.COMPONENT_PRIORITY, Integer.toString(priority))); - resCompOptTriples.addAll(Arrays.asList(compName, - ResourceKeys.COMPONENT_INSTANCES, Long.toString(numInstances))); - resCompOptTriples.addAll(Arrays.asList(compName, ResourceKeys.YARN_MEMORY, - memory)); - resCompOptTriples.addAll(Arrays.asList(compName, ResourceKeys.YARN_CORES, - cpus.toString())); - if (component.getPlacementPolicy() != null) { - resCompOptTriples.addAll(Arrays.asList(compName, - ResourceKeys.COMPONENT_PLACEMENT_POLICY, - component.getPlacementPolicy().getLabel())); - } - - return resCompOptTriples; - } - - private static UserGroupInformation getSliderUser() { - if (SLIDER_USER != null) { - return SLIDER_USER; - } - UserGroupInformation sliderUser = null; - UserGroupInformation.setConfiguration(SLIDER_CONFIG); - String loggedInUser = getUserToRunAs(); - try { - sliderUser = UserGroupInformation.getBestUGI(null, loggedInUser); - // TODO: Once plugged into RM process we should remove the previous call - // and replace it with getCurrentUser as commented below. - // sliderUser = UserGroupInformation.getCurrentUser(); - } catch (IOException e) { - throw new RuntimeException("Unable to create UGI (slider user)", e); - } - return sliderUser; - } - - private T invokeSliderClientRunnable( - final SliderClientContextRunnable runnable) - throws IOException, InterruptedException, YarnException { - try { - T value = SLIDER_USER.doAs(new PrivilegedExceptionAction() { - @Override - public T run() throws Exception { - return runnable.run(SLIDER_CLIENT); - } - }); - return value; - } catch (UndeclaredThrowableException e) { - Throwable cause = e.getCause(); - if (cause instanceof YarnException) { - YarnException ye = (YarnException) cause; - throw ye; - } - throw e; - } - } - protected static SliderClient createSliderClient() { if (SLIDER_CLIENT != null) { return SLIDER_CLIENT; } - org.apache.hadoop.conf.Configuration sliderClientConfiguration = SLIDER_CONFIG; + org.apache.hadoop.conf.Configuration sliderClientConfiguration = + SLIDER_CONFIG; SliderClient client = new SliderClient() { - @Override - public void init(org.apache.hadoop.conf.Configuration conf) { + @Override public void init(org.apache.hadoop.conf.Configuration conf) { super.init(conf); try { initHadoopBinding(); - } catch (SliderException e) { - throw new RuntimeException( - "Unable to automatically init Hadoop binding", e); - } catch (IOException e) { + } catch (SliderException | IOException e) { throw new RuntimeException( "Unable to automatically init Hadoop binding", e); } @@ -730,8 +144,7 @@ public class ApplicationApiService implements ApplicationApi { try { logger .debug("Slider Client configuration: {}", sliderClientConfiguration); - sliderClientConfiguration = client.bindArgs(sliderClientConfiguration, - new String[] { "help" }); + sliderClientConfiguration = client.bindArgs(sliderClientConfiguration, new String[] { "help" }); client.init(sliderClientConfiguration); client.start(); } catch (Exception e) { @@ -741,608 +154,116 @@ public class ApplicationApiService implements ApplicationApi { return client; } - private static String getUserToRunAs() { - String user = System.getenv(PROPERTY_APP_RUNAS_USER); - if (StringUtils.isEmpty(user)) { - user = "root"; - } - return user; - } + // The information this REST endpoint currently returned can be retrieved from + // RM web services + // Probably the data from AM is more important. Do that later. +// @GET @Consumes({ MediaType.APPLICATION_JSON }) +// @Produces({ MediaType.APPLICATION_JSON }) +// public Response getApplications(@QueryParam("state") String state) { +// logger.info("GET: getApplications with param state = {}", state); +// return null; +// } - private static org.apache.hadoop.conf.Configuration getSliderClientConfiguration() { - if (SLIDER_CONFIG != null) { - return SLIDER_CONFIG; - } - YarnConfiguration yarnConfig = new YarnConfiguration(); - logger.info("prop yarn.resourcemanager.address = {}", - yarnConfig.get("yarn.resourcemanager.address")); - - return yarnConfig; - } - - private interface SliderClientContextRunnable { - T run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException; - } - - @GET - @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON }) - public Response getApplications(@QueryParam("state") String state) { - logger.info("GET: getApplications with param state = {}", state); - - // Get all applications in a specific state - lighter projection. For full - // detail, call getApplication on a specific app. - Set applications; - try { - if (StringUtils.isNotEmpty(state)) { - ApplicationStatus appStatus = new ApplicationStatus(); - try { - ApplicationState.valueOf(state); - } catch (IllegalArgumentException e) { - appStatus.setDiagnostics("Invalid value for param state - " + state); - return Response.status(Status.BAD_REQUEST).entity(appStatus).build(); - } - applications = getSliderApplications(state); - } else { - applications = getSliderApplications(true); - } - } catch (Exception e) { - logger.error("Get applications failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } - - Set apps = new HashSet(); - if (applications.size() > 0) { - try { - for (ApplicationReport app : applications) { - Application application = new Application(); - application.setLifetime(app.getApplicationTimeouts().get( - ApplicationTimeoutType.LIFETIME).getRemainingTime()); - application.setLaunchTime(new Date(app.getStartTime())); - application.setName(app.getName()); - // Containers not required, setting to null to avoid empty list - application.setContainers(null); - apps.add(application); - } - } catch (Exception e) { - logger.error("Get applications failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } - } - - return Response.ok().entity(apps).build(); - } - - @GET - @Path("/{app_name}") + @GET @Path("/{app_name}") @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response getApplication(@PathParam("app_name") String appName) { logger.info("GET: getApplication for appName = {}", appName); + ApplicationStatus applicationStatus = new ApplicationStatus(); // app name validation if (!SliderUtils.isClusternameValid(appName)) { - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Invalid application name"); + applicationStatus.setDiagnostics("Invalid application name: " + appName); applicationStatus.setCode(ERROR_CODE_APP_NAME_INVALID); return Response.status(Status.NOT_FOUND).entity(applicationStatus) .build(); } - // Check if app exists try { - int livenessCheck = getSliderList(appName); - if (livenessCheck < 0) { - logger.info("Application not running"); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics(ERROR_APPLICATION_NOT_RUNNING); - applicationStatus.setCode(ERROR_CODE_APP_IS_NOT_RUNNING); + Application app = SLIDER_CLIENT.actionStatus(appName); + ApplicationReport report = SLIDER_CLIENT.findInstance(appName); + if (app != null && report != null) { + app.setLifetime( + report.getApplicationTimeouts().get(ApplicationTimeoutType.LIFETIME) + .getRemainingTime()); + logger.info("Application = {}", app); + return Response.ok(app).build(); + } else { + String message = "Application " + appName + " does not exist."; + logger.info(message); + applicationStatus.setCode(ERROR_CODE_APP_DOES_NOT_EXIST); + applicationStatus.setDiagnostics(message); return Response.status(Status.NOT_FOUND).entity(applicationStatus) .build(); } - } catch (UnknownApplicationInstanceException e) { - logger.error("Get application failed, application not found", e); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics(ERROR_APPLICATION_DOES_NOT_EXIST); - applicationStatus.setCode(ERROR_CODE_APP_DOES_NOT_EXIST); - return Response.status(Status.NOT_FOUND).entity(applicationStatus) - .build(); - } catch (Exception e) { - logger.error("Get application failed, application not running", e); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics(ERROR_APPLICATION_NOT_RUNNING); - applicationStatus.setCode(ERROR_CODE_APP_IS_NOT_RUNNING); - return Response.status(Status.NOT_FOUND).entity(applicationStatus) - .build(); - } - - Application app = new Application(); - app.setName(appName); - app.setUri(CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH + "/" - + appName); - // TODO: add status - app.setState(ApplicationState.ACCEPTED); - JsonObject appStatus = null; - JsonObject appRegistryQuicklinks = null; - try { - appStatus = getSliderApplicationStatus(appName); - appRegistryQuicklinks = getSliderApplicationRegistry(appName, - "quicklinks"); - return populateAppData(app, appStatus, appRegistryQuicklinks); - } catch (BadClusterStateException | NotFoundException e) { - logger.error( - "Get application failed, application not in running state yet", e); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Application not running yet"); - applicationStatus.setCode(ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET); - return Response.status(Status.NOT_FOUND).entity(applicationStatus) - .build(); } catch (Exception e) { logger.error("Get application failed", e); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Failed to retrieve application: " - + e.getMessage()); + applicationStatus + .setDiagnostics("Failed to retrieve application: " + e.getMessage()); return Response.status(Status.INTERNAL_SERVER_ERROR) .entity(applicationStatus).build(); } } - private Response populateAppData(Application app, JsonObject appStatus, - JsonObject appRegistryQuicklinks) { - String appName = jsonGetAsString(appStatus, "name"); - Long totalNumberOfRunningContainers = 0L; - Long totalExpectedNumberOfRunningContainers = 0L; - Long totalNumberOfIpAssignedContainers = 0L; - - // info - JsonObject applicationInfo = jsonGetAsObject(appStatus, "info"); - if (applicationInfo != null) { - String applicationId = jsonGetAsString(applicationInfo, "info.am.app.id"); - if (applicationId != null) { - app.setId(applicationId); - } - } - - // state - String appState = jsonGetAsString(appStatus, "state"); - if (appState == null) { - // consider that app is still in ACCEPTED state - appState = String.valueOf(StateValues.STATE_INCOMPLETE); - } - switch (Integer.parseInt(appState)) { - case StateValues.STATE_LIVE: - app.setState(ApplicationState.STARTED); - break; - case StateValues.STATE_CREATED: - case StateValues.STATE_INCOMPLETE: - case StateValues.STATE_SUBMITTED: - app.setState(ApplicationState.ACCEPTED); - return Response.ok(app).build(); - case StateValues.STATE_DESTROYED: - case StateValues.STATE_STOPPED: - app.setState(ApplicationState.STOPPED); - return Response.ok(app).build(); - default: - break; - } - - // start time - app.setLaunchTime(appStatus.get("createTime") == null ? null - : new Date(appStatus.get("createTime").getAsLong())); - - app.setLifetime(queryLifetime(appName)); - - // Quicklinks - Map appQuicklinks = new HashMap<>(); - for (Map.Entry quicklink : appRegistryQuicklinks - .entrySet()) { - appQuicklinks.put(quicklink.getKey(), quicklink.getValue() == null ? null - : quicklink.getValue().getAsString()); - } - if (!appQuicklinks.isEmpty()) { - app.setQuicklinks(appQuicklinks); - } - - ArrayList componentNames = new ArrayList<>(); - - // status.live - JsonObject applicationStatus = jsonGetAsObject(appStatus, "status"); - // roles - JsonObject applicationRoles = jsonGetAsObject(appStatus, "roles"); - // statistics - JsonObject applicationStatistics = jsonGetAsObject(appStatus, "statistics"); - if (applicationRoles == null) { - // initialize to empty object to avoid too many null checks - applicationRoles = EMPTY_JSON_OBJECT; - } - if (applicationStatus != null) { - JsonObject applicationLive = jsonGetAsObject(applicationStatus, "live"); - if (applicationLive != null) { - for (Entry entry : applicationLive.entrySet()) { - if (entry.getKey().equals(SLIDER_APPMASTER_COMPONENT_NAME)) { - continue; - } - componentNames.add(entry.getKey()); - JsonObject componentRole = applicationRoles - .get(entry.getKey()) == null ? EMPTY_JSON_OBJECT - : applicationRoles.get(entry.getKey()).getAsJsonObject(); - JsonObject liveContainers = entry.getValue().getAsJsonObject(); - if (liveContainers != null) { - for (Map.Entry liveContainerEntry : liveContainers - .entrySet()) { - String containerId = liveContainerEntry.getKey(); - Container container = new Container(); - container.setId(containerId); - JsonObject liveContainer = (JsonObject) liveContainerEntry - .getValue(); - container - .setLaunchTime(liveContainer.get("startTime") == null ? null - : new Date(liveContainer.get("startTime").getAsLong())); - container - .setComponentName(jsonGetAsString(liveContainer, "role")); - container.setIp(jsonGetAsString(liveContainer, "ip")); - // If ip is non-null increment count - if (container.getIp() != null) { - totalNumberOfIpAssignedContainers++; - } - container.setHostname(jsonGetAsString(liveContainer, "hostname")); - container.setState(ContainerState.INIT); - if (StringUtils.isNotEmpty(container.getIp()) - && StringUtils.isNotEmpty(container.getHostname())) { - container.setState(ContainerState.READY); - } - container.setBareHost(jsonGetAsString(liveContainer, "host")); - container.setUri(CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH - + "/" + appName + CONTAINERS_API_RESOURCE_PATH + "/" - + containerId); - Resource resource = new Resource(); - resource.setCpus(jsonGetAsInt(componentRole, "yarn.vcores")); - resource.setMemory(jsonGetAsString(componentRole, "yarn.memory")); - container.setResource(resource); - Artifact artifact = new Artifact(); - String dockerImageName = jsonGetAsString(componentRole, - "docker.image"); - if (StringUtils.isNotEmpty(dockerImageName)) { - artifact.setId(dockerImageName); - artifact.setType(Artifact.TypeEnum.DOCKER); - } else { - // Might have to handle tarballs here - artifact.setType(null); - } - container.setArtifact(artifact); - container.setPrivilegedContainer( - jsonGetAsBoolean(componentRole, "docker.usePrivileged")); - // TODO: add container property - for response only? - app.addContainer(container); - } - } - } - } - } - - // application info - if (applicationRoles != null && !componentNames.isEmpty()) { - JsonObject applicationRole = jsonGetAsObject(applicationRoles, - componentNames.get(0)); - if (applicationRole != null) { - Artifact artifact = new Artifact(); - // how to get artifact id - docker image name?? - artifact.setId(null); - } - } - - // actual and expected number of containers - if (applicationStatistics != null) { - for (Entry entry : applicationStatistics.entrySet()) { - if (entry.getKey().equals(SLIDER_APPMASTER_COMPONENT_NAME)) { - continue; - } - JsonObject containerStats = (JsonObject) entry.getValue(); - totalNumberOfRunningContainers += jsonGetAsInt(containerStats, - "containers.live"); - totalExpectedNumberOfRunningContainers += jsonGetAsInt(containerStats, - "containers.desired"); - } - app.setNumberOfContainers(totalExpectedNumberOfRunningContainers); - app.setNumberOfRunningContainers(totalNumberOfRunningContainers); - } - - // If all containers of the app has IP assigned, then according to the REST - // API it is considered to be READY. Note, application readiness from - // end-users point of view, is out of scope of the REST API. Also, this - // readiness has nothing to do with readiness-check defined at the component - // level (which is used for dependency resolution of component DAG). - if (totalNumberOfIpAssignedContainers - .longValue() == totalExpectedNumberOfRunningContainers.longValue()) { - app.setState(ApplicationState.READY); - } - logger.info("Application = {}", app); - return Response.ok(app).build(); - } - - private String jsonGetAsString(JsonObject object, String key) { - return object.get(key) == null ? null : object.get(key).getAsString(); - } - - private Integer jsonGetAsInt(JsonObject object, String key) { - return object.get(key) == null ? null - : object.get(key).isJsonNull() ? null : object.get(key).getAsInt(); - } - - private Boolean jsonGetAsBoolean(JsonObject object, String key) { - return object.get(key) == null ? null - : object.get(key).isJsonNull() ? null : object.get(key).getAsBoolean(); - } - - private JsonObject jsonGetAsObject(JsonObject object, String key) { - return object.get(key) == null ? null : object.get(key).getAsJsonObject(); - } - - private long queryLifetime(String appName) { - try { - return invokeSliderClientRunnable( - new SliderClientContextRunnable() { - @Override - public Long run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException { - ApplicationReport report = sliderClient.findInstance(appName); - return report.getApplicationTimeouts() - .get(ApplicationTimeoutType.LIFETIME).getRemainingTime(); - } - }); - } catch (Exception e) { - logger.error("Error when querying lifetime for " + appName, e); - return DEFAULT_UNLIMITED_LIFETIME; - } - } - - private JsonObject getSliderApplicationStatus(final String appName) - throws IOException, YarnException, InterruptedException { - - return invokeSliderClientRunnable( - new SliderClientContextRunnable() { - @Override - public JsonObject run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException { - String status = null; - try { - status = sliderClient.actionStatus(appName); - } catch (BadClusterStateException e) { - logger.warn("Application not running yet", e); - return EMPTY_JSON_OBJECT; - } catch (Exception e) { - logger.error("Exception calling slider.actionStatus", e); - return EMPTY_JSON_OBJECT; - } - JsonElement statusElement = JSON_PARSER.parse(status); - return (statusElement == null || statusElement instanceof JsonNull) - ? EMPTY_JSON_OBJECT : (JsonObject) statusElement; - } - }); - } - - private JsonObject getSliderApplicationRegistry(final String appName, - final String registryName) - throws IOException, YarnException, InterruptedException { - final ActionRegistryArgs registryArgs = new ActionRegistryArgs(); - registryArgs.name = appName; - registryArgs.getConf = registryName; - registryArgs.format = ConfigFormat.JSON.toString(); - - return invokeSliderClientRunnable( - new SliderClientContextRunnable() { - @Override - public JsonObject run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException { - String registry = null; - try { - registry = sliderClient.actionRegistryGetConfig(registryArgs) - .asJson(); - } catch (FileNotFoundException | NotFoundException e) { - // ignore and return empty object - return EMPTY_JSON_OBJECT; - } catch (Exception e) { - logger.error("Exception calling slider.actionRegistryGetConfig", - e); - return EMPTY_JSON_OBJECT; - } - JsonElement registryElement = JSON_PARSER.parse(registry); - return (registryElement == null - || registryElement instanceof JsonNull) ? EMPTY_JSON_OBJECT - : (JsonObject) registryElement; - } - }); - } - - private Integer getSliderList(final String appName) - throws IOException, YarnException, InterruptedException { - return getSliderList(appName, true); - } - - private Integer getSliderList(final String appName, final boolean liveOnly) - throws IOException, YarnException, InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public Integer run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - int status = 0; - if (liveOnly) { - status = sliderClient.actionList(appName); - } else { - status = sliderClient.actionList(appName, ACTION_LIST_ARGS); - } - return status; - } - }); - } - - private Set getSliderApplications(final String state) - throws IOException, YarnException, InterruptedException { - return getSliderApplications(false, state); - } - - private Set getSliderApplications(final boolean liveOnly) - throws IOException, YarnException, InterruptedException { - return getSliderApplications(liveOnly, null); - } - - private Set getSliderApplications(final boolean liveOnly, - final String state) - throws IOException, YarnException, InterruptedException { - return invokeSliderClientRunnable( - new SliderClientContextRunnable>() { - @Override - public Set run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException { - Set apps; - ActionListArgs listArgs = new ActionListArgs(); - if (liveOnly) { - apps = sliderClient.getApplicationList(null); - } else if (StringUtils.isNotEmpty(state)) { - listArgs.state = state; - apps = sliderClient.getApplicationList(null, listArgs); - } else { - apps = sliderClient.getApplicationList(null, listArgs); - } - return apps; - } - }); - } - @DELETE @Path("/{app_name}") @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response deleteApplication(@PathParam("app_name") String appName) { logger.info("DELETE: deleteApplication for appName = {}", appName); + return stopApplication(appName, true); + } + private Response stopApplication(String appName, boolean destroy) { try { - Response stopResponse = stopSliderApplication(appName); - if (stopResponse.getStatus() == Status.INTERNAL_SERVER_ERROR - .getStatusCode()) { - return Response.status(Status.NOT_FOUND).build(); + SLIDER_CLIENT.actionStop(appName, ACTION_FREEZE_ARGS); + if (destroy) { + SLIDER_CLIENT.actionDestroy(appName); + logger.info("Successfully deleted application {}", appName); + } else { + logger.info("Successfully stopped application {}", appName); } - } catch (UnknownApplicationInstanceException e) { - logger.error("Application does not exist", e); - return Response.status(Status.NOT_FOUND).build(); + return Response.status(Status.NO_CONTENT).build(); + } catch (ApplicationNotFoundException e) { + ApplicationStatus applicationStatus = new ApplicationStatus(); + applicationStatus.setDiagnostics( + "Application " + appName + " not found " + e.getMessage()); + return Response.status(Status.NOT_FOUND).entity(applicationStatus) + .build(); } catch (Exception e) { - logger.error("Delete application failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); + ApplicationStatus applicationStatus = new ApplicationStatus(); + applicationStatus.setDiagnostics(e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(applicationStatus).build(); } + } - // Although slider client stop returns immediately, it usually takes a - // little longer for it to stop from YARN point of view. Slider destroy - // fails if the application is not completely stopped. Hence the need to - // call destroy in a controlled loop few times (only if exit code is - // EXIT_APPLICATION_IN_USE or EXIT_INSTANCE_EXISTS), before giving up. - boolean keepTrying = true; - int maxDeleteAttempts = 5; - int deleteAttempts = 0; - int sleepIntervalInMillis = 500; - while (keepTrying && deleteAttempts < maxDeleteAttempts) { - try { - destroySliderApplication(appName); - keepTrying = false; - } catch (SliderException e) { - if (e.getExitCode() == SliderExitCodes.EXIT_APPLICATION_IN_USE - || e.getExitCode() == SliderExitCodes.EXIT_INSTANCE_EXISTS) { - deleteAttempts++; - // If we used up all the allowed delete attempts, let's log it as - // error before giving up. Otherwise log as warn. - if (deleteAttempts < maxDeleteAttempts) { - logger.warn("Application not in stopped state, waiting for {}ms" - + " before trying delete again", sleepIntervalInMillis); - } else { - logger.error("Delete application failed", e); - } - try { - Thread.sleep(sleepIntervalInMillis); - } catch (InterruptedException e1) { - } - } else { - logger.error("Delete application threw exception", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } - } catch (Exception e) { - logger.error("Delete application failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } + @PUT @Path("/{app_name}/components/{component_name}") + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response updateComponent(@PathParam("app_name") String appName, + @PathParam("component_name") String componentName, Component component) { + + if (component.getNumberOfContainers() < 0) { + return Response.status(Status.BAD_REQUEST).entity( + "Application = " + appName + ", Component = " + component.getName() + + ": Invalid number of containers specified " + component + .getNumberOfContainers()).build(); + } + try { + long original = SLIDER_CLIENT.flex(appName, component); + return Response.ok().entity( + "Updating " + componentName + " size from " + original + " to " + + component.getNumberOfContainers()).build(); + } catch (YarnException | IOException e) { + ApplicationStatus status = new ApplicationStatus(); + status.setDiagnostics(e.getMessage()); + return Response.status(Status.INTERNAL_SERVER_ERROR).entity(status) + .build(); } - return Response.status(Status.NO_CONTENT).build(); } - private Response stopSliderApplication(final String appName) - throws IOException, YarnException, InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public Response run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - int returnCode = sliderClient.actionFreeze(appName, ACTION_FREEZE_ARGS); - if (returnCode == 0) { - logger.info("Successfully stopped application {}", appName); - return Response.status(Status.NO_CONTENT).build(); - } else { - logger.error("Stop of application {} failed with return code ", - appName, returnCode); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Stop of application " + appName - + " failed"); - return Response.status(Status.INTERNAL_SERVER_ERROR) - .entity(applicationStatus).build(); - } - } - }); - } - - private Response startSliderApplication(final String appName, Application app) - throws IOException, YarnException, InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public Response run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - ActionThawArgs thawArgs = new ActionThawArgs(); - if (app.getLifetime() == null) { - app.setLifetime(DEFAULT_UNLIMITED_LIFETIME); - } - thawArgs.lifetime = app.getLifetime(); - int returnCode = sliderClient.actionThaw(appName, thawArgs); - if (returnCode == 0) { - logger.info("Successfully started application {}", appName); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setState(ApplicationState.ACCEPTED); - applicationStatus.setUri(CONTEXT_ROOT - + APPLICATIONS_API_RESOURCE_PATH + "/" + appName); - // 202 = ACCEPTED - return Response.status(HTTP_STATUS_CODE_ACCEPTED) - .entity(applicationStatus).build(); - } else { - logger.error("Start of application {} failed with returnCode ", - appName, returnCode); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Start of application " + appName - + " failed"); - return Response.status(Status.INTERNAL_SERVER_ERROR) - .entity(applicationStatus).build(); - } - } - }); - } - - private Void destroySliderApplication(final String appName) - throws IOException, YarnException, InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public Void run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - sliderClient.actionDestroy(appName); - return null; - } - }); - } - - @PUT - @Path("/{app_name}") + @PUT @Path("/{app_name}") @Consumes({ MediaType.APPLICATION_JSON }) @Produces({ MediaType.APPLICATION_JSON }) public Response updateApplication(@PathParam("app_name") String appName, @@ -1354,158 +275,70 @@ public class ApplicationApiService implements ApplicationApi { // path param updateAppData.setName(appName); - // Adding support for stop and start // For STOP the app should be running. If already stopped then this // operation will be a no-op. For START it should be in stopped state. // If already running then this operation will be a no-op. - - // Check if app exists in any state - try { - int appsFound = getSliderList(appName, false); - if (appsFound < 0) { - return Response.status(Status.NOT_FOUND).build(); - } - } catch (Exception e) { - logger.error("Update application failed", e); - return Response.status(Status.NOT_FOUND).build(); - } - - // If a STOP is requested if (updateAppData.getState() != null && updateAppData.getState() == ApplicationState.STOPPED) { - try { - int livenessCheck = getSliderList(appName); - if (livenessCheck == 0) { - return stopSliderApplication(appName); - } else { - logger.info("Application {} is already stopped", appName); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Application " + appName - + " is already stopped"); - return Response.status(Status.BAD_REQUEST).entity(applicationStatus) - .build(); - } - } catch (Exception e) { - logger.error("Stop application failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } + return stopApplication(appName, false); } // If a START is requested if (updateAppData.getState() != null && updateAppData.getState() == ApplicationState.STARTED) { - try { - int livenessCheck = getSliderList(appName); - if (livenessCheck != 0) { - return startSliderApplication(appName, updateAppData); - } else { - logger.info("Application {} is already running", appName); - ApplicationStatus applicationStatus = new ApplicationStatus(); - applicationStatus.setDiagnostics("Application " + appName - + " is already running"); - applicationStatus.setUri(CONTEXT_ROOT - + APPLICATIONS_API_RESOURCE_PATH + "/" + appName); - return Response.status(Status.BAD_REQUEST).entity(applicationStatus) - .build(); - } - } catch (Exception e) { - logger.error("Start application failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } - } - - // If no of instances specified then treat it as a flex - if (updateAppData.getNumberOfContainers() != null - && updateAppData.getComponents() == null) { - updateAppData.setComponents(getDefaultComponentAsList()); - } - - // At this point if there are components then it is a flex - if (updateAppData.getComponents() != null) { - try { - int livenessCheck = getSliderList(appName); - if (livenessCheck == 0) { - flexSliderApplication(appName, updateAppData); - } - return Response.status(Status.NO_CONTENT).build(); - } catch (Exception e) { - logger.error("Update application failed", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } + return startApplication(appName); } // If new lifetime value specified then update it if (updateAppData.getLifetime() != null && updateAppData.getLifetime() > 0) { - try { - updateAppLifetime(appName, updateAppData.getLifetime()); - } catch (Exception e) { - logger.error("Failed to update application (" + appName + ") lifetime (" - + updateAppData.getLifetime() + ")", e); - return Response.status(Status.INTERNAL_SERVER_ERROR).build(); - } + return updateLifetime(appName, updateAppData); + } + + // flex a single component app + if (updateAppData.getNumberOfContainers() != null && !ServiceApiUtil + .hasComponent( + updateAppData)) { + Component defaultComp = ServiceApiUtil.createDefaultComponent(updateAppData); + return updateComponent(updateAppData.getName(), defaultComp.getName(), + defaultComp); } // If nothing happens consider it a no-op return Response.status(Status.NO_CONTENT).build(); } - private Void updateAppLifetime(String appName, long lifetime) - throws InterruptedException, YarnException, IOException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override public Void run(SliderClient sliderClient) - throws YarnException, IOException, InterruptedException { - ActionUpdateArgs args = new ActionUpdateArgs(); - args.lifetime = lifetime; - sliderClient.actionUpdate(appName, args); - return null; + private Response updateLifetime(String appName, Application updateAppData) { + try { + String newLifeTime = + SLIDER_CLIENT.updateLifetime(appName, updateAppData.getLifetime()); + return Response.ok("Application " + appName + " lifeTime is successfully updated to " + + updateAppData.getLifetime() + " seconds from now: " + newLifeTime).build(); + } catch (Exception e) { + String message = + "Failed to update application (" + appName + ") lifetime (" + + updateAppData.getLifetime() + ")"; + logger.error(message, e); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(message + " : " + e.getMessage()).build(); + } + } + + private Response startApplication(String appName) { + try { + int ret = SLIDER_CLIENT.actionList(appName); + if (ret == 0) { + return Response.ok() + .entity("Application " + appName + " is already alive.").build(); } - }); - } - - // create default component and initialize with app level global values - private List getDefaultComponentAsList(Application app) { - List comps = getDefaultComponentAsList(); - Component comp = comps.get(0); - comp.setArtifact(app.getArtifact()); - comp.setResource(app.getResource()); - comp.setNumberOfContainers(app.getNumberOfContainers()); - comp.setLaunchCommand(app.getLaunchCommand()); - return comps; - } - - private List getDefaultComponentAsList() { - Component comp = new Component(); - comp.setName(DEFAULT_COMPONENT_NAME); - List comps = new ArrayList<>(); - comps.add(comp); - return comps; - } - - private Void flexSliderApplication(final String appName, - final Application updateAppData) throws IOException, YarnException, - InterruptedException { - return invokeSliderClientRunnable(new SliderClientContextRunnable() { - @Override - public Void run(SliderClient sliderClient) throws YarnException, - IOException, InterruptedException { - ActionFlexArgs flexArgs = new ActionFlexArgs(); - ComponentArgsDelegate compDelegate = new ComponentArgsDelegate(); - Long globalNumberOfContainers = updateAppData.getNumberOfContainers(); - for (Component comp : updateAppData.getComponents()) { - Long noOfContainers = comp.getNumberOfContainers() == null - ? globalNumberOfContainers : comp.getNumberOfContainers(); - if (noOfContainers != null) { - compDelegate.componentTuples.addAll( - Arrays.asList(comp.getName(), String.valueOf(noOfContainers))); - } - } - if (!compDelegate.componentTuples.isEmpty()) { - flexArgs.componentDelegate = compDelegate; - sliderClient.actionFlex(appName, flexArgs); - } - return null; - } - }); + SLIDER_CLIENT.actionStart(appName, null); + logger.info("Successfully started application " + appName); + return Response.ok("Application " + appName + " is successfully started").build(); + } catch (Exception e) { + String message = "Failed to start application " + appName; + logger.info(message, e); + return Response.status(Status.INTERNAL_SERVER_ERROR) + .entity(message + ": " + e.getMessage()).build(); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java index e1bddb5fcfb..7fc01a1f2a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java @@ -17,7 +17,7 @@ package org.apache.hadoop.yarn.services.webapp; -import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*; +import static org.apache.slider.util.RestApiConstants.*; import java.io.IOException; import java.net.InetAddress; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java index 7bfb410e301..abad34e9f7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java @@ -17,16 +17,15 @@ package org.apache.hadoop.yarn.services.api.impl; -import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*; -import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*; +import static org.apache.slider.util.RestApiConstants.*; +import static org.apache.slider.util.RestApiErrorMessages.*; -import java.util.HashMap; -import java.util.Map; +import java.util.ArrayList; import org.apache.slider.api.resource.Application; import org.apache.slider.api.resource.Artifact; import org.apache.slider.api.resource.Resource; -import org.apache.slider.common.SliderKeys; +import org.apache.slider.util.ServiceApiUtil; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -61,12 +60,10 @@ public class TestApplicationApiService { @Test(timeout = 90000) public void testValidateApplicationPostPayload() throws Exception { Application app = new Application(); - Map compNameArtifactIdMap = new HashMap<>(); // no name try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with no name"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage()); @@ -77,8 +74,7 @@ public class TestApplicationApiService { for (String badName : badNames) { app.setName(badName); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID_FORMAT, @@ -89,8 +85,7 @@ public class TestApplicationApiService { // no artifact app.setName("finance_home"); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with no artifact"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_ARTIFACT_INVALID, e.getMessage()); @@ -100,8 +95,7 @@ public class TestApplicationApiService { Artifact artifact = new Artifact(); app.setArtifact(artifact); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with no artifact id"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage()); @@ -112,8 +106,7 @@ public class TestApplicationApiService { artifact.setId("app.io/hbase:facebook_0.2"); app.setNumberOfContainers(5l); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); } catch (IllegalArgumentException e) { logger.error("application attributes specified should be valid here", e); Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); @@ -124,22 +117,18 @@ public class TestApplicationApiService { Assert.assertEquals(app.getComponents().get(0).getName(), DEFAULT_COMPONENT_NAME); Assert.assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME); - Assert.assertEquals("Property not set", - app.getConfiguration().getProperties() - .get(SliderKeys.COMPONENT_TYPE_KEY), - SliderKeys.COMPONENT_TYPE_EXTERNAL_APP); + //TODO handle external app // unset artifact type, default component and no of containers to test other // validation logic artifact.setType(null); - app.setComponents(null); + app.setComponents(new ArrayList<>()); app.setNumberOfContainers(null); // resource not specified artifact.setId("docker.io/centos:centos7"); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with no resource"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_RESOURCE_INVALID, e.getMessage()); @@ -149,28 +138,18 @@ public class TestApplicationApiService { Resource res = new Resource(); app.setResource(res); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with no memory"); } catch (IllegalArgumentException e) { Assert.assertEquals(ERROR_RESOURCE_MEMORY_INVALID, e.getMessage()); } - // cpus not specified - res.setMemory("2gb"); - try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); - Assert.fail(EXCEPTION_PREFIX + "application with no cpu"); - } catch (IllegalArgumentException e) { - Assert.assertEquals(ERROR_RESOURCE_CPUS_INVALID, e.getMessage()); - } - + // cpu does not need to be always specified, it's an optional feature in yarn // invalid no of cpus + res.setMemory("100mb"); res.setCpus(-2); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail( EXCEPTION_PREFIX + "application with invalid no of cpups"); } catch (IllegalArgumentException e) { @@ -180,8 +159,7 @@ public class TestApplicationApiService { // number of containers not specified res.setCpus(2); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail( EXCEPTION_PREFIX + "application with no container count"); } catch (IllegalArgumentException e) { @@ -191,8 +169,7 @@ public class TestApplicationApiService { // specifying profile along with cpus/memory raises exception res.setProfile("hbase_finance_large"); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with resource profile along with cpus/memory"); } catch (IllegalArgumentException e) { @@ -205,8 +182,7 @@ public class TestApplicationApiService { res.setCpus(null); res.setMemory(null); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); Assert.fail(EXCEPTION_PREFIX + "application with resource profile only - NOT SUPPORTED"); } catch (IllegalArgumentException e) { @@ -222,8 +198,7 @@ public class TestApplicationApiService { // everything valid here app.setNumberOfContainers(5l); try { - appApiService.validateApplicationPostPayload(app, - compNameArtifactIdMap); + ServiceApiUtil.validateApplicationPostPayload(app); } catch (IllegalArgumentException e) { logger.error("application attributes specified should be valid here", e); Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java index d21785f5663..f6a2cc90205 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java @@ -59,14 +59,6 @@ public interface SliderApplicationApi { */ ConfTreeOperations getDesiredResources() throws IOException; - /** - * Put an updated resources structure. This triggers a cluster flex - * operation - * @param updated updated resources - * @throws IOException on any problem. - */ - void putDesiredResources(ConfTree updated) throws IOException; - /** * Get the aggregate resolved model * @return the aggregate configuration of what was asked for diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java index 893e706fdfa..f384927eb9f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java @@ -52,12 +52,9 @@ public interface SliderClusterProtocol extends VersionedProtocol { Messages.UpgradeContainersRequestProto request) throws IOException, YarnException; - /** - * Flex the cluster. - */ - Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request) - throws IOException; + Messages.FlexComponentResponseProto flexComponent( + Messages.FlexComponentRequestProto request) throws IOException; /** * Get the current cluster status @@ -120,13 +117,6 @@ public interface SliderClusterProtocol extends VersionedProtocol { Messages.AMSuicideResponseProto amSuicide(Messages.AMSuicideRequestProto request) throws IOException; - /** - * Get the instance definition - */ - Messages.GetInstanceDefinitionResponseProto getInstanceDefinition( - Messages.GetInstanceDefinitionRequestProto request) - throws IOException, YarnException; - /** * Get the application liveness * @return current liveness information diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java index cc3355a2778..502b5193791 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Application.java @@ -28,6 +28,7 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -55,11 +56,11 @@ public class Application extends BaseResource { private Long numberOfRunningContainers = null; private Long lifetime = null; private PlacementPolicy placementPolicy = null; - private List components = null; - private Configuration configuration = null; + private List components = new ArrayList<>(); + private Configuration configuration = new Configuration(); private List containers = new ArrayList<>(); private ApplicationState state = null; - private Map quicklinks = null; + private Map quicklinks = new HashMap<>(); private String queue = null; /** @@ -285,6 +286,15 @@ public class Application extends BaseResource { this.components = components; } + public Component getComponent(String name) { + for (Component component : components) { + if (component.getName().equals(name)) { + return component; + } + } + return null; + } + /** * Config properties of an application. Configurations provided at the * application/global level are available to all the components. Specific diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java index 4f50564ee0d..e7f3796b07a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Component.java @@ -22,6 +22,7 @@ import io.swagger.annotations.ApiModelProperty; import java.io.Serializable; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; @@ -49,15 +50,17 @@ public class Component implements Serializable { private String name = null; private List dependencies = new ArrayList(); private ReadinessCheck readinessCheck = null; - private Artifact artifact = null; + private Artifact artifact = new Artifact(); private String launchCommand = null; - private Resource resource = null; + private Resource resource = new Resource(); private Long numberOfContainers = null; - private Boolean uniqueComponentSupport = null; - private Boolean runPrivilegedContainer = null; + private Boolean uniqueComponentSupport = false; + private Boolean runPrivilegedContainer = false; private PlacementPolicy placementPolicy = null; - private Configuration configuration = null; + private Configuration configuration = new Configuration(); private List quicklinks = new ArrayList(); + private List containers = + Collections.synchronizedList(new ArrayList()); /** * Name of the application component (mandatory). @@ -196,6 +199,29 @@ public class Component implements Serializable { this.numberOfContainers = numberOfContainers; } + @ApiModelProperty(example = "null", value = "Containers of a started component. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.") + @JsonProperty("containers") + public List getContainers() { + return containers; + } + + public void setContainers(List containers) { + this.containers = containers; + } + + public void addContainer(Container container) { + this.containers.add(container); + } + + public Container getContainer(String id) { + for (Container container : containers) { + if (container.getId().equals(id)) { + return container; + } + } + return null; + } + /** * Certain applications need to define multiple components using the same * artifact and resource profile, differing only in configurations. In such @@ -354,6 +380,8 @@ public class Component implements Serializable { sb.append(" resource: ").append(toIndentedString(resource)).append("\n"); sb.append(" numberOfContainers: ") .append(toIndentedString(numberOfContainers)).append("\n"); + sb.append(" containers: ").append(toIndentedString(containers)) + .append("\n"); sb.append(" uniqueComponentSupport: ") .append(toIndentedString(uniqueComponentSupport)).append("\n"); sb.append(" runPrivilegedContainer: ") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java index bad68c16a6d..cdc96b81a07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/ConfigFile.java @@ -21,6 +21,7 @@ import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import java.io.Serializable; +import java.util.Map; import java.util.Objects; import javax.xml.bind.annotation.XmlElement; @@ -62,7 +63,7 @@ public class ConfigFile implements Serializable { private TypeEnum type = null; private String destFile = null; private String srcFile = null; - private Object props = null; + private Map props = null; /** * Config file in the standard format like xml, properties, json, yaml, @@ -104,6 +105,8 @@ public class ConfigFile implements Serializable { } /** + * TODO this probably is not required for non-template configs. It is now used as symlink for localization for non-template configs - we could infer the name from destFile instead + * * Required for type template. This provides the source location of the * template which needs to be mounted as dest_file post property * substitutions. Typically the src_file would point to a source controlled @@ -131,21 +134,36 @@ public class ConfigFile implements Serializable { * src_file is mandatory and the src_file content is dumped to dest_file post * property substitutions. **/ - public ConfigFile props(Object props) { + public ConfigFile props(Map props) { this.props = props; return this; } @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If the type is template then the attribute src_file is mandatory and the src_file content is dumped to dest_file post property substitutions.") @JsonProperty("props") - public Object getProps() { + public Map getProps() { return props; } - public void setProps(Object props) { + public void setProps(Map props) { this.props = props; } + public long getLong(String name, long defaultValue) { + if (name == null) { + return defaultValue; + } + String value = props.get(name.trim()); + return Long.parseLong(value); + } + + public boolean getBoolean(String name, boolean defaultValue) { + if (name == null) { + return defaultValue; + } + return Boolean.valueOf(props.get(name.trim())); + } + @Override public boolean equals(java.lang.Object o) { if (this == o) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java index c4f2ad4fb55..c43bd64dbeb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java @@ -29,6 +29,7 @@ import java.util.Objects; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.commons.lang.StringUtils; /** * Set of configuration properties that can be injected into the application @@ -104,6 +105,35 @@ public class Configuration implements Serializable { this.files = files; } + public long getPropertyLong(String name, long defaultValue) { + if (name == null) { + return defaultValue; + } + String value = properties.get(name.trim()); + if (StringUtils.isEmpty(value)) { + return defaultValue; + } + return Long.parseLong(value); + } + + public String getProperty(String name, String defaultValue) { + if (name == null) { + return defaultValue; + } + return properties.get(name.trim()); + } + + public void setProperty(String name, String value) { + properties.put(name, value); + } + + public String getProperty(String name) { + if (name == null) { + return null; + } + return properties.get(name.trim()); + } + @Override public boolean equals(java.lang.Object o) { if (this == o) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java index 190121d4031..c2553692b0a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Resource.java @@ -39,7 +39,7 @@ public class Resource extends BaseResource implements Cloneable { private static final long serialVersionUID = -6431667797380250037L; private String profile = null; - private Integer cpus = null; + private Integer cpus = 1; private String memory = null; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 00e2b62c691..f4ea70b3703 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -20,7 +20,6 @@ package org.apache.slider.client; import com.google.common.annotations.VisibleForTesting; import com.google.common.io.Files; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; @@ -35,7 +34,6 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.registry.client.api.RegistryConstants; import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.registry.client.binding.RegistryPathUtils; @@ -55,40 +53,44 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationTimeout; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Times; -import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterNode; import org.apache.slider.api.SliderApplicationApi; import org.apache.slider.api.SliderClusterProtocol; -import org.apache.slider.api.StateValues; import org.apache.slider.api.proto.Messages; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.api.types.NodeInformationList; -import org.apache.slider.api.types.SliderInstanceDescription; import org.apache.slider.client.ipc.SliderApplicationIpcClient; import org.apache.slider.client.ipc.SliderClusterOperations; import org.apache.slider.common.Constants; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.params.AbstractActionArgs; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; import org.apache.slider.common.params.ActionAMSuicideArgs; import org.apache.slider.common.params.ActionClientArgs; import org.apache.slider.common.params.ActionCreateArgs; import org.apache.slider.common.params.ActionDependencyArgs; -import org.apache.slider.common.params.ActionDestroyArgs; import org.apache.slider.common.params.ActionDiagnosticArgs; import org.apache.slider.common.params.ActionEchoArgs; import org.apache.slider.common.params.ActionExistsArgs; @@ -113,20 +115,13 @@ import org.apache.slider.common.params.ActionUpgradeArgs; import org.apache.slider.common.params.Arguments; import org.apache.slider.common.params.ClientArgs; import org.apache.slider.common.params.CommonArgs; -import org.apache.slider.common.params.LaunchArgsAccessor; import org.apache.slider.common.tools.ConfigHelper; -import org.apache.slider.common.tools.Duration; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.common.tools.SliderVersionInfo; -import org.apache.slider.core.buildutils.InstanceBuilder; import org.apache.slider.core.buildutils.InstanceIO; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTree; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.conf.MapOperations; -import org.apache.slider.core.conf.ResourcesInputPropertiesValidator; -import org.apache.slider.core.conf.TemplateInputPropertiesValidator; import org.apache.slider.core.exceptions.BadClusterStateException; import org.apache.slider.core.exceptions.BadCommandArgumentsException; import org.apache.slider.core.exceptions.BadConfigException; @@ -137,18 +132,13 @@ import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.exceptions.UnknownApplicationInstanceException; import org.apache.slider.core.exceptions.UsageException; import org.apache.slider.core.exceptions.WaitTimeoutException; -import org.apache.slider.core.launch.AppMasterLauncher; import org.apache.slider.core.launch.ClasspathConstructor; import org.apache.slider.core.launch.CredentialUtils; import org.apache.slider.core.launch.JavaCommandLineBuilder; -import org.apache.slider.core.launch.LaunchedApplication; import org.apache.slider.core.launch.SerializedApplicationReport; import org.apache.slider.core.main.RunService; -import org.apache.slider.core.persist.AppDefinitionPersister; import org.apache.slider.core.persist.ApplicationReportSerDeser; -import org.apache.slider.core.persist.ConfPersister; import org.apache.slider.core.persist.JsonSerDeser; -import org.apache.slider.core.persist.LockAcquireFailedException; import org.apache.slider.core.registry.SliderRegistryUtils; import org.apache.slider.core.registry.YarnAppListClient; import org.apache.slider.core.registry.docstore.ConfigFormat; @@ -160,19 +150,19 @@ import org.apache.slider.core.registry.docstore.PublishedExportsSet; import org.apache.slider.core.registry.retrieve.RegistryRetriever; import org.apache.slider.core.zk.BlockingZKWatcher; import org.apache.slider.core.zk.ZKIntegration; -import org.apache.slider.core.zk.ZKPathBuilder; import org.apache.slider.providers.AbstractClientProvider; +import org.apache.slider.providers.ProviderUtils; import org.apache.slider.providers.SliderProviderFactory; import org.apache.slider.providers.agent.AgentKeys; -import org.apache.slider.providers.docker.DockerClientProvider; -import org.apache.slider.providers.slideram.SliderAMClientProvider; import org.apache.slider.server.appmaster.SliderAppMaster; import org.apache.slider.server.appmaster.rpc.RpcBinder; import org.apache.slider.server.services.utility.AbstractSliderLaunchedService; +import org.apache.slider.util.ServiceApiUtil; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.data.ACL; +import org.codehaus.jackson.map.PropertyNamingStrategy; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; @@ -182,7 +172,6 @@ import java.io.ByteArrayOutputStream; import java.io.Console; import java.io.File; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InterruptedIOException; @@ -191,10 +180,7 @@ import java.io.PrintStream; import java.io.PrintWriter; import java.io.StringWriter; import java.io.Writer; -import java.net.InetSocketAddress; -import java.net.URISyntaxException; import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -213,14 +199,11 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.apache.hadoop.registry.client.binding.RegistryUtils.*; -import static org.apache.slider.api.InternalKeys.*; -import static org.apache.slider.api.OptionKeys.*; -import static org.apache.slider.api.ResourceKeys.*; +import static org.apache.slider.api.InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH; import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG; import static org.apache.slider.common.params.SliderActions.*; import static org.apache.slider.common.tools.SliderUtils.*; - /** * Client service for Slider */ @@ -246,6 +229,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe public static final String E_NO_RESOURCE_MANAGER = "No valid Resource Manager address provided"; public static final String E_PACKAGE_EXISTS = "Package exists"; private static PrintStream clientOutputStream = System.out; + private static final JsonSerDeser jsonSerDeser = + new JsonSerDeser(Application.class, + PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); // value should not be changed without updating string find in slider.py private static final String PASSWORD_PROMPT = "Enter password for"; @@ -362,16 +348,22 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe serviceArgs.getActionAMSuicideArgs()); break; - case ACTION_BUILD: - exitCode = actionBuild(clusterName, serviceArgs.getActionBuildArgs()); - break; - case ACTION_CLIENT: exitCode = actionClient(serviceArgs.getActionClientArgs()); break; case ACTION_CREATE: - exitCode = actionCreate(clusterName, serviceArgs.getActionCreateArgs()); + ActionCreateArgs args = serviceArgs.getActionCreateArgs(); + File file = args.getAppDef(); + Path filePath = new Path(file.getAbsolutePath()); + log.info("Loading app definition from: " + filePath); + Application application = + jsonSerDeser.load(FileSystem.getLocal(getConfig()), filePath); + if(args.lifetime > 0) { + application.setLifetime(args.lifetime); + } + application.setName(clusterName); + actionCreate(application); break; case ACTION_DEPENDENCY: @@ -379,7 +371,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe break; case ACTION_DESTROY: - exitCode = actionDestroy(clusterName, serviceArgs.getActionDestroyArgs()); + actionDestroy(clusterName); break; case ACTION_DIAGNOSTICS: @@ -392,11 +384,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe break; case ACTION_FLEX: - exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs()); + actionFlex(clusterName, serviceArgs.getActionFlexArgs()); break; - case ACTION_FREEZE: - exitCode = actionFreeze(clusterName, serviceArgs.getActionFreezeArgs()); + case ACTION_STOP: + actionStop(clusterName, serviceArgs.getActionFreezeArgs()); break; case ACTION_HELP: @@ -456,8 +448,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe exitCode = actionStatus(clusterName, serviceArgs.getActionStatusArgs()); break; - case ACTION_THAW: - exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs()); + case ACTION_START: + exitCode = actionStart(clusterName, serviceArgs.getActionThawArgs()); break; case ACTION_TOKENS: @@ -516,7 +508,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe String zkPath = ZKIntegration.mkClusterPath(user, clusterName); Exception e = null; try { - Configuration config = getConfig(); ZKIntegration client = getZkClient(clusterName, user); if (client != null) { if (client.exists(zkPath)) { @@ -627,76 +618,31 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * force=true by default. */ @Override - public int actionDestroy(String clustername) throws YarnException, - IOException { - ActionDestroyArgs destroyArgs = new ActionDestroyArgs(); - destroyArgs.force = true; - return actionDestroy(clustername, destroyArgs); - } - - @Override - public int actionDestroy(String clustername, - ActionDestroyArgs destroyArgs) throws YarnException, IOException { - // verify that a live cluster isn't there - validateClusterName(clustername); - //no=op, it is now mandatory. - verifyBindingsDefined(); - verifyNoLiveClusters(clustername, "Destroy"); - boolean forceDestroy = destroyArgs.force; - log.debug("actionDestroy({}, force={})", clustername, forceDestroy); - - // create the directory path - Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); - // delete the directory; + public void actionDestroy(String appName) + throws YarnException, IOException { + validateClusterName(appName); + Path appDir = sliderFileSystem.buildClusterDirPath(appName); FileSystem fs = sliderFileSystem.getFileSystem(); - boolean exists = fs.exists(clusterDirectory); - if (exists) { - log.debug("Application Instance {} found at {}: destroying", clustername, clusterDirectory); - if (!forceDestroy) { - // fail the command if --force is not explicitly specified - throw new UsageException("Destroy will permanently delete directories and registries. " - + "Reissue this command with the --force option if you want to proceed."); + if (fs.exists(appDir)) { + if (fs.delete(appDir, true)) { + log.info("Successfully deleted application + " + appName); + return; + } else { + String message = + "Failed to delete application + " + appName + " at: " + appDir; + log.info(message); + throw new YarnException(message); } - if (!fs.delete(clusterDirectory, true)) { - log.warn("Filesystem returned false from delete() operation"); - } - - if(!deleteZookeeperNode(clustername)) { - log.warn("Unable to perform node cleanup in Zookeeper."); - } - - if (fs.exists(clusterDirectory)) { - log.warn("Failed to delete {}", clusterDirectory); - } - - } else { - log.debug("Application Instance {} already destroyed", clustername); } - - // rm the registry entry —do not let this block the destroy operations - String registryPath = SliderRegistryUtils.registryPathForInstance( - clustername); - try { - getRegistryOperations().delete(registryPath, true); - } catch (IOException e) { - log.warn("Error deleting registry entry {}: {} ", registryPath, e, e); - } catch (SliderException e) { - log.warn("Error binding to registry {} ", e, e); + if (!deleteZookeeperNode(appName)) { + String message = + "Failed to cleanup cleanup application " + appName + " in zookeeper"; + log.warn(message); + throw new YarnException(message); } - - List instances = findAllLiveInstances(clustername); - // detect any race leading to cluster creation during the check/destroy process - // and report a problem. - if (!instances.isEmpty()) { - throw new SliderException(EXIT_APPLICATION_IN_USE, - clustername + ": " - + E_DESTROY_CREATE_RACE_CONDITION - + " :" + - instances.get(0)); - } - log.info("Destroyed cluster {}", clustername); - return EXIT_SUCCESS; + //TODO clean registry } + @Override public int actionAmSuicide(String clustername, @@ -715,203 +661,285 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return factory.createClientProvider(); } - /** - * Create the cluster -saving the arguments to a specification file first - * @param clustername cluster name - * @return the status code - * @throws YarnException Yarn problems - * @throws IOException other problems - * @throws BadCommandArgumentsException bad arguments. - */ - public int actionCreate(String clustername, ActionCreateArgs createArgs) throws - YarnException, - IOException { - actionBuild(clustername, createArgs); - Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); - AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( - clustername, clusterDirectory); - try { - checkForCredentials(getConfig(), instanceDefinition.getAppConf(), - clustername); - } catch (IOException e) { - sliderFileSystem.getFileSystem().delete(clusterDirectory, true); - throw e; + public ApplicationId actionCreate(Application application) + throws IOException, YarnException { + ServiceApiUtil.validateApplicationPostPayload(application); + String appName = application.getName(); + validateClusterName(appName); + verifyNoLiveApp(appName, "Create"); + Path appDir = checkAppNotExistOnHdfs(application); + + ApplicationId appId = submitApp(application); + application.setId(appId.toString()); + // write app definition on to hdfs + persistApp(appDir, application); + return appId; + //TODO deal with registry + } + + private ApplicationId submitApp(Application app) + throws IOException, YarnException { + String appName = app.getName(); + Configuration conf = getConfig(); + Path appRootDir = sliderFileSystem.buildClusterDirPath(app.getName()); + deployedClusterName = appName; + + YarnClientApplication yarnApp = yarnClient.createApplication(); + ApplicationSubmissionContext submissionContext = + yarnApp.getApplicationSubmissionContext(); + applicationId = submissionContext.getApplicationId(); + submissionContext.setKeepContainersAcrossApplicationAttempts(true); + if (app.getLifetime() > 0) { + Map appTimeout = new HashMap<>(); + appTimeout.put(ApplicationTimeoutType.LIFETIME, app.getLifetime()); + submissionContext.setApplicationTimeouts(appTimeout); + } + submissionContext.setMaxAppAttempts(conf.getInt(KEY_AM_RESTART_LIMIT, 2)); + + Map localResources = + new HashMap(); + + // copy local slideram-log4j.properties to hdfs and add to localResources + boolean hasSliderAMLog4j = + addAMLog4jResource(appName, conf, localResources); + // copy jars to hdfs and add to localResources + Path tempPath = addJarResource(appName, localResources); + // add keytab if in secure env + addKeytabResourceIfSecure(sliderFileSystem, localResources, conf, appName); + printLocalResources(localResources); + + //TODO SliderAMClientProvider#copyEnvVars + //TODO localResource putEnv + + Map env = addAMEnv(conf, tempPath); + + // create AM CLI + String cmdStr = + buildCommandLine(appName, conf, appRootDir, hasSliderAMLog4j); + + //TODO set log aggregation context + //TODO set retry window + submissionContext.setResource(Resource.newInstance( + conf.getLong(KEY_AM_RESOURCE_MEM, DEFAULT_KEY_AM_RESOURCE_MEM), 1)); + submissionContext.setQueue(conf.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE)); + submissionContext.setApplicationName(appName); + submissionContext.setApplicationType(SliderKeys.APP_TYPE); + Set appTags = + AbstractClientProvider.createApplicationTags(appName, null, null); + if (!appTags.isEmpty()) { + submissionContext.setApplicationTags(appTags); + } + ContainerLaunchContext amLaunchContext = + Records.newRecord(ContainerLaunchContext.class); + amLaunchContext.setCommands(Collections.singletonList(cmdStr)); + amLaunchContext.setEnvironment(env); + amLaunchContext.setLocalResources(localResources); + addCredentialsIfSecure(conf, amLaunchContext); + submissionContext.setAMContainerSpec(amLaunchContext); + yarnClient.submitApplication(submissionContext); + return submissionContext.getApplicationId(); + } + + private void printLocalResources(Map map) { + log.info("Added LocalResource for localization: "); + StringBuilder builder = new StringBuilder(); + for (Map.Entry entry : map.entrySet()) { + builder.append(entry.getKey()).append(" -> ") + .append(entry.getValue().getResource().getFile()) + .append(System.lineSeparator()); + } + log.info(builder.toString()); + } + + private void addCredentialsIfSecure(Configuration conf, + ContainerLaunchContext amLaunchContext) throws IOException { + if (UserGroupInformation.isSecurityEnabled()) { + // pick up oozie credentials + Credentials credentials = + CredentialUtils.loadTokensFromEnvironment(System.getenv(), conf); + if (credentials == null) { + // nothing from oozie, so build up directly + credentials = new Credentials( + UserGroupInformation.getCurrentUser().getCredentials()); + CredentialUtils.addRMRenewableFSDelegationTokens(conf, + sliderFileSystem.getFileSystem(), credentials); + } else { + log.info("Using externally supplied credentials to launch AM"); + } + amLaunchContext.setTokens(CredentialUtils.marshallCredentials(credentials)); + } + } + + private String buildCommandLine(String appName, Configuration conf, + Path appRootDir, boolean hasSliderAMLog4j) throws BadConfigException { + JavaCommandLineBuilder CLI = new JavaCommandLineBuilder(); + CLI.forceIPv4().headless(); + //TODO CLI.setJVMHeap + //TODO CLI.addJVMOPTS + if (hasSliderAMLog4j) { + CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME); + CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR); + } + CLI.add(SliderAppMaster.SERVICE_CLASSNAME); + CLI.add(ACTION_CREATE, appName); + //TODO debugAM CLI.add(Arguments.ARG_DEBUG) + CLI.add(Arguments.ARG_CLUSTER_URI, appRootDir.toUri()); +// InetSocketAddress rmSchedulerAddress = getRmSchedulerAddress(conf); +// String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress); +// CLI.add(Arguments.ARG_RM_ADDR, rmAddr); + // pass the registry binding + CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT, + RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT); + CLI.addMandatoryConfOption(conf, RegistryConstants.KEY_REGISTRY_ZK_QUORUM); + if(isHadoopClusterSecure(conf)) { + //TODO Is this required ?? + // if the cluster is secure, make sure that + // the relevant security settings go over + CLI.addConfOption(conf, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); + } +// // copy over any/all YARN RM client values, in case the server-side XML conf file +// // has the 0.0.0.0 address +// CLI.addConfOptions(conf, YarnConfiguration.RM_ADDRESS, +// YarnConfiguration.RM_CLUSTER_ID, YarnConfiguration.RM_HOSTNAME, +// YarnConfiguration.RM_PRINCIPAL); + + // write out the path output + CLI.addOutAndErrFiles(STDOUT_AM, STDERR_AM); + String cmdStr = CLI.build(); + log.info("Completed setting up app master command: {}", cmdStr); + return cmdStr; + } + + private Map addAMEnv(Configuration conf, Path tempPath) + throws IOException { + Map env = new HashMap(); + ClasspathConstructor classpath = + buildClasspath(SliderKeys.SUBMITTED_CONF_DIR, "lib", + sliderFileSystem, getUsingMiniMRCluster()); + env.put("CLASSPATH", classpath.buildClasspath()); + env.put("LANG", "en_US.UTF-8"); + env.put("LC_ALL", "en_US.UTF-8"); + env.put("LANGUAGE", "en_US.UTF-8"); + String jaas = System.getenv(HADOOP_JAAS_DEBUG); + if (jaas != null) { + env.put(HADOOP_JAAS_DEBUG, jaas); + } + env.putAll(getAmLaunchEnv(conf)); + log.info("AM env: \n{}", stringifyMap(env)); + return env; + } + + private Path addJarResource(String appName, + Map localResources) + throws IOException, SliderException { + Path libPath = sliderFileSystem.buildClusterDirPath(appName); + ProviderUtils + .addProviderJar(localResources, SliderAppMaster.class, SLIDER_JAR, + sliderFileSystem, libPath, "lib", false); + Path dependencyLibTarGzip = sliderFileSystem.getDependencyTarGzip(); + if (sliderFileSystem.isFile(dependencyLibTarGzip)) { + log.info("Loading lib tar from " + sliderFileSystem.getFileSystem() + .getScheme() + ": " + dependencyLibTarGzip); + SliderUtils.putAmTarGzipAndUpdate(localResources, sliderFileSystem); + } else { + String[] libs = SliderUtils.getLibDirs(); + log.info("Loading dependencies from local file system: " + Arrays + .toString(libs)); + for (String libDirProp : libs) { + ProviderUtils + .addAllDependencyJars(localResources, sliderFileSystem, libPath, + "lib", libDirProp); + } + } + return libPath; + } + + private boolean addAMLog4jResource(String appName, Configuration conf, + Map localResources) + throws IOException, BadClusterStateException { + boolean hasSliderAMLog4j = false; + String hadoopConfDir = + System.getenv(ApplicationConstants.Environment.HADOOP_CONF_DIR.name()); + if (hadoopConfDir != null) { + File localFile = + new File(hadoopConfDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME); + if (localFile.exists()) { + Path localFilePath = createLocalPath(localFile); + Path appDirPath = sliderFileSystem.buildClusterDirPath(appName); + Path remoteConfPath = + new Path(appDirPath, SliderKeys.SUBMITTED_CONF_DIR); + Path remoteFilePath = + new Path(remoteConfPath, SliderKeys.LOG4J_SERVER_PROP_FILENAME); + copy(conf, localFilePath, remoteFilePath); + LocalResource localResource = sliderFileSystem + .createAmResource(remoteConfPath, LocalResourceType.FILE); + localResources.put(localFilePath.getName(), localResource); + hasSliderAMLog4j = true; + } + } + return hasSliderAMLog4j; + } + + private Path checkAppNotExistOnHdfs(Application application) + throws IOException, SliderException { + Path appDir = sliderFileSystem.buildClusterDirPath(application.getName()); + sliderFileSystem.verifyDirectoryNonexistent( + new Path(appDir, application.getName() + ".json")); + return appDir; + } + + private void persistApp(Path appDir, Application application) + throws IOException, SliderException { + FsPermission appDirPermission = new FsPermission("777"); + sliderFileSystem.createWithPermissions(appDir, appDirPermission); + Path appJson = new Path(appDir, application.getName() + ".json"); + jsonSerDeser + .save(sliderFileSystem.getFileSystem(), appJson, application, true); + log.info( + "Persisted application " + application.getName() + " at " + appJson); + } + + private void addKeytabResourceIfSecure(SliderFileSystem fileSystem, + Map localResource, Configuration conf, + String appName) throws IOException, BadConfigException { + if (!UserGroupInformation.isSecurityEnabled()) { + return; + } + String keytabPreInstalledOnHost = + conf.get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); + if (StringUtils.isEmpty(keytabPreInstalledOnHost)) { + String amKeytabName = + conf.get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); + String keytabDir = conf.get(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); + Path keytabPath = + fileSystem.buildKeytabPath(keytabDir, amKeytabName, appName); + if (fileSystem.getFileSystem().exists(keytabPath)) { + LocalResource keytabRes = + fileSystem.createAmResource(keytabPath, LocalResourceType.FILE); + localResource + .put(SliderKeys.KEYTAB_DIR + "/" + amKeytabName, keytabRes); + log.info("Adding AM keytab on hdfs: " + keytabPath); + } else { + log.warn("No keytab file was found at {}.", keytabPath); + if (conf.getBoolean(KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) { + throw new BadConfigException("No keytab file was found at %s.", + keytabPath); + } else { + log.warn("The AM will be " + + "started without a kerberos authenticated identity. " + + "The application is therefore not guaranteed to remain " + + "operational beyond 24 hours."); + } + } } - return startCluster(clustername, createArgs, createArgs.lifetime); } @Override public int actionUpgrade(String clustername, ActionUpgradeArgs upgradeArgs) throws YarnException, IOException { - File template = upgradeArgs.template; - File resources = upgradeArgs.resources; - List containers = upgradeArgs.containers; - List components = upgradeArgs.components; - - // For upgrade spec, let's be little more strict with validation. If either - // --template or --resources is specified, then both needs to be specified. - // Otherwise the internal app config and resources states of the app will be - // unwantedly modified and the change will take effect to the running app - // immediately. - require(!(template != null && resources == null), - "Option %s must be specified with option %s", - Arguments.ARG_RESOURCES, Arguments.ARG_TEMPLATE); - - require(!(resources != null && template == null), - "Option %s must be specified with option %s", - Arguments.ARG_TEMPLATE, Arguments.ARG_RESOURCES); - - // For upgrade spec, both --template and --resources should be specified - // and neither of --containers or --components should be used - if (template != null && resources != null) { - require(CollectionUtils.isEmpty(containers), - "Option %s cannot be specified with %s or %s", - Arguments.ARG_CONTAINERS, Arguments.ARG_TEMPLATE, - Arguments.ARG_RESOURCES); - require(CollectionUtils.isEmpty(components), - "Option %s cannot be specified with %s or %s", - Arguments.ARG_COMPONENTS, Arguments.ARG_TEMPLATE, - Arguments.ARG_RESOURCES); - - // not an error to try to upgrade a stopped cluster, just return success - // code, appropriate log messages have already been dumped - if (!isAppInRunningState(clustername)) { - return EXIT_SUCCESS; - } - - // Now initiate the upgrade spec flow - buildInstanceDefinition(clustername, upgradeArgs, true, true, true); - SliderClusterOperations clusterOperations = createClusterOperations(clustername); - clusterOperations.amSuicide("AM restarted for application upgrade", 1, 1000); - return EXIT_SUCCESS; - } - - // Since neither --template or --resources were specified, it is upgrade - // containers flow. Here any one or both of --containers and --components - // can be specified. If a container is specified with --containers option - // and also belongs to a component type specified with --components, it will - // be upgraded only once. - return actionUpgradeContainers(clustername, upgradeArgs); - } - - private int actionUpgradeContainers(String clustername, - ActionUpgradeArgs upgradeArgs) throws YarnException, IOException { - verifyBindingsDefined(); - validateClusterName(clustername); - int waittime = upgradeArgs.getWaittime(); // ignored for now - String text = "Upgrade containers"; - log.debug("actionUpgradeContainers({}, reason={}, wait={})", clustername, - text, waittime); - - // not an error to try to upgrade a stopped cluster, just return success - // code, appropriate log messages have already been dumped - if (!isAppInRunningState(clustername)) { - return EXIT_SUCCESS; - } - - // Create sets of containers and components to get rid of duplicates and - // for quick lookup during checks below - Set containers = new HashSet<>(); - if (upgradeArgs.containers != null) { - containers.addAll(new ArrayList<>(upgradeArgs.containers)); - } - Set components = new HashSet<>(); - if (upgradeArgs.components != null) { - components.addAll(new ArrayList<>(upgradeArgs.components)); - } - - // check validity of component names and running containers here - List liveContainers = getContainers(clustername); - Set validContainers = new HashSet<>(); - Set validComponents = new HashSet<>(); - for (ContainerInformation liveContainer : liveContainers) { - boolean allContainersAndComponentsAccountedFor = true; - if (CollectionUtils.isNotEmpty(containers)) { - if (containers.contains(liveContainer.containerId)) { - containers.remove(liveContainer.containerId); - validContainers.add(liveContainer.containerId); - } - allContainersAndComponentsAccountedFor = false; - } - if (CollectionUtils.isNotEmpty(components)) { - if (components.contains(liveContainer.component)) { - components.remove(liveContainer.component); - validComponents.add(liveContainer.component); - } - allContainersAndComponentsAccountedFor = false; - } - if (allContainersAndComponentsAccountedFor) { - break; - } - } - - // If any item remains in containers or components then they are invalid. - // Log warning for them and proceed. - if (CollectionUtils.isNotEmpty(containers)) { - log.warn("Invalid set of containers provided {}", containers); - } - if (CollectionUtils.isNotEmpty(components)) { - log.warn("Invalid set of components provided {}", components); - } - - // If not a single valid container or component is specified do not proceed - if (CollectionUtils.isEmpty(validContainers) - && CollectionUtils.isEmpty(validComponents)) { - log.error("Not a single valid container or component specified. Nothing to do."); - return EXIT_NOT_FOUND; - } - - SliderClusterProtocol appMaster = connect(findInstance(clustername)); - Messages.UpgradeContainersRequestProto r = - Messages.UpgradeContainersRequestProto - .newBuilder() - .setMessage(text) - .addAllContainer(validContainers) - .addAllComponent(validComponents) - .build(); - appMaster.upgradeContainers(r); - log.info("Cluster upgrade issued for -"); - if (CollectionUtils.isNotEmpty(validContainers)) { - log.info(" Containers (total {}): {}", validContainers.size(), - validContainers); - } - if (CollectionUtils.isNotEmpty(validComponents)) { - log.info(" Components (total {}): {}", validComponents.size(), - validComponents); - } - - return EXIT_SUCCESS; - } - - // returns true if and only if app is in RUNNING state - private boolean isAppInRunningState(String clustername) throws YarnException, - IOException { - // is this actually a known cluster? - sliderFileSystem.locateInstanceDefinition(clustername); - ApplicationReport app = findInstance(clustername); - if (app == null) { - // exit early - log.info("Cluster {} not running", clustername); - return false; - } - log.debug("App to upgrade was found: {}:\n{}", clustername, - new OnDemandReportStringifier(app)); - if (app.getYarnApplicationState().ordinal() >= YarnApplicationState.FINISHED.ordinal()) { - log.info("Cluster {} is in a terminated state {}. Use command '{}' instead.", - clustername, app.getYarnApplicationState(), ACTION_UPDATE); - return false; - } - - // IPC request to upgrade containers is possible if the app is running. - if (app.getYarnApplicationState().ordinal() < YarnApplicationState.RUNNING - .ordinal()) { - log.info("Cluster {} is in a pre-running state {}. To upgrade it needs " - + "to be RUNNING.", clustername, app.getYarnApplicationState()); - return false; - } - - return true; + //TODO + return 0; } protected static void checkForCredentials(Configuration conf, @@ -952,15 +980,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } } - private static char[] readOnePassword(String alias) throws IOException { - Console console = System.console(); - if (console == null) { - throw new IOException("Unable to input password for " + alias + - " because System.console() is null"); - } - return readPassword(alias, console); - } - private static char[] readPassword(String alias, Console console) throws IOException { char[] cred = null; @@ -986,16 +1005,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return cred; } - @Override - public int actionBuild(String clustername, - AbstractClusterBuildingActionArgs buildInfo) throws - YarnException, - IOException { - - buildInstanceDefinition(clustername, buildInfo, false, false); - return EXIT_SUCCESS; - } - @Override public int actionKeytab(ActionKeytabArgs keytabInfo) throws YarnException, IOException { @@ -1527,12 +1536,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe if (buildInfo.lifetime > 0) { updateLifetime(clustername, buildInfo.lifetime); } else { - buildInstanceDefinition(clustername, buildInfo, true, true); + //TODO upgrade } return EXIT_SUCCESS; } - public void updateLifetime(String appName, long lifetime) + public String updateLifetime(String appName, long lifetime) throws YarnException, IOException { EnumSet appStates = EnumSet.range( YarnApplicationState.NEW, YarnApplicationState.RUNNING); @@ -1553,396 +1562,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe log.info("Successfully updated lifetime for an application: appName = " + appName + ", appId = " + appId + ". New expiry time in ISO8601 format is " + newTimeout); - } - - /** - * Build up the AggregateConfiguration for an application instance then - * persists it - * @param clustername name of the cluster - * @param buildInfo the arguments needed to build the cluster - * @param overwrite true if existing cluster directory can be overwritten - * @param liveClusterAllowed true if live cluster can be modified - * @throws YarnException - * @throws IOException - */ - - public void buildInstanceDefinition(String clustername, - AbstractClusterBuildingActionArgs buildInfo, boolean overwrite, - boolean liveClusterAllowed) throws YarnException, IOException { - buildInstanceDefinition(clustername, buildInfo, overwrite, - liveClusterAllowed, false); - } - - public void buildInstanceDefinition(String clustername, - AbstractClusterBuildingActionArgs buildInfo, boolean overwrite, - boolean liveClusterAllowed, boolean isUpgradeFlow) throws YarnException, - IOException { - // verify that a live cluster isn't there - validateClusterName(clustername); - verifyBindingsDefined(); - if (!liveClusterAllowed) { - verifyNoLiveClusters(clustername, "Create"); - } - - Configuration conf = getConfig(); - String registryQuorum = lookupZKQuorum(); - - Path appconfdir = buildInfo.getConfdir(); - // Provider - String providerName = buildInfo.getProvider(); - requireArgumentSet(Arguments.ARG_PROVIDER, providerName); - log.debug("Provider is {}", providerName); - SliderAMClientProvider sliderAM = new SliderAMClientProvider(conf); - AbstractClientProvider provider = - createClientProvider(providerName); - InstanceBuilder builder = - new InstanceBuilder(sliderFileSystem, - getConfig(), - clustername); - - AggregateConf instanceDefinition = new AggregateConf(); - ConfTreeOperations appConf = instanceDefinition.getAppConfOperations(); - ConfTreeOperations resources = instanceDefinition.getResourceOperations(); - ConfTreeOperations internal = instanceDefinition.getInternalOperations(); - //initial definition is set by the providers - sliderAM.prepareInstanceConfiguration(instanceDefinition); - provider.prepareInstanceConfiguration(instanceDefinition); - - //load in any specified on the command line - if (buildInfo.resources != null) { - try { - resources.mergeFile(buildInfo.resources, - new ResourcesInputPropertiesValidator()); - - } catch (IOException e) { - throw new BadConfigException(e, - "incorrect argument to %s: \"%s\" : %s ", - Arguments.ARG_RESOURCES, - buildInfo.resources, - e.toString()); - } - } - if (buildInfo.template != null) { - try { - appConf.mergeFile(buildInfo.template, - new TemplateInputPropertiesValidator()); - } catch (IOException e) { - throw new BadConfigException(e, - "incorrect argument to %s: \"%s\" : %s ", - Arguments.ARG_TEMPLATE, - buildInfo.template, - e.toString()); - } - } - - if (isUpgradeFlow) { - ActionUpgradeArgs upgradeInfo = (ActionUpgradeArgs) buildInfo; - if (!upgradeInfo.force) { - validateClientAndClusterResource(clustername, resources); - } - } - - //get the command line options - ConfTree cmdLineAppOptions = buildInfo.buildAppOptionsConfTree(); - ConfTree cmdLineResourceOptions = buildInfo.buildResourceOptionsConfTree(); - - appConf.merge(cmdLineAppOptions); - - AppDefinitionPersister appDefinitionPersister = new AppDefinitionPersister(sliderFileSystem); - appDefinitionPersister.processSuppliedDefinitions(clustername, buildInfo, appConf); - - // put the role counts into the resources file - Map argsRoleMap = buildInfo.getComponentMap(); - for (Map.Entry roleEntry : argsRoleMap.entrySet()) { - String count = roleEntry.getValue(); - String key = roleEntry.getKey(); - log.info("{} => {}", key, count); - resources.getOrAddComponent(key).put(COMPONENT_INSTANCES, count); - } - - //all CLI role options - Map> appOptionMap = - buildInfo.getCompOptionMap(); - appConf.mergeComponents(appOptionMap); - - //internal picks up core. values only - internal.propagateGlobalKeys(appConf, "slider."); - internal.propagateGlobalKeys(appConf, "internal."); - - //copy over role. and yarn. values ONLY to the resources - if (PROPAGATE_RESOURCE_OPTION) { - resources.propagateGlobalKeys(appConf, "component."); - resources.propagateGlobalKeys(appConf, "role."); - resources.propagateGlobalKeys(appConf, "yarn."); - resources.mergeComponentsPrefix(appOptionMap, "component.", true); - resources.mergeComponentsPrefix(appOptionMap, "yarn.", true); - resources.mergeComponentsPrefix(appOptionMap, "role.", true); - } - - // resource component args - appConf.merge(cmdLineResourceOptions); - resources.merge(cmdLineResourceOptions); - resources.mergeComponents(buildInfo.getResourceCompOptionMap()); - - builder.init(providerName, instanceDefinition); - builder.resolve(); - builder.propagateFilename(); - builder.propagatePrincipals(); - builder.setImageDetailsIfAvailable(buildInfo.getImage(), - buildInfo.getAppHomeDir()); - builder.setQueue(buildInfo.queue); - - String quorum = buildInfo.getZKhosts(); - if (isUnset(quorum)) { - quorum = registryQuorum; - } - if (isUnset(quorum)) { - throw new BadConfigException(E_NO_ZOOKEEPER_QUORUM); - } - ZKPathBuilder zkPaths = new ZKPathBuilder(getAppName(), - getUsername(), - clustername, - registryQuorum, - quorum); - String zookeeperRoot = buildInfo.getAppZKPath(); - - if (isSet(zookeeperRoot)) { - zkPaths.setAppPath(zookeeperRoot); - } else { - String createDefaultZkNode = appConf.getGlobalOptions() - .getOption(AgentKeys.CREATE_DEF_ZK_NODE, "false"); - if (createDefaultZkNode.equals("true")) { - String defaultZKPath = createZookeeperNode(clustername, false); - log.debug("ZK node created for application instance: {}", defaultZKPath); - if (defaultZKPath != null) { - zkPaths.setAppPath(defaultZKPath); - } - } else { - // create AppPath if default is being used - String defaultZKPath = createZookeeperNode(clustername, true); - log.debug("ZK node assigned to application instance: {}", defaultZKPath); - zkPaths.setAppPath(defaultZKPath); - } - } - - builder.addZKBinding(zkPaths); - - //then propagate any package URI - if (buildInfo.packageURI != null) { - appConf.set(AgentKeys.PACKAGE_PATH, buildInfo.packageURI); - } - - propagatePythonExecutable(conf, instanceDefinition); - - // make any substitutions needed at this stage - replaceTokens(appConf.getConfTree(), getUsername(), clustername); - - // TODO: Refactor the validation code and persistence code - try { - persistInstanceDefinition(overwrite, appconfdir, builder); - appDefinitionPersister.persistPackages(); - - } catch (LockAcquireFailedException e) { - log.warn("Failed to get a Lock on {} : {}", builder, e, e); - throw new BadClusterStateException("Failed to save " + clustername - + ": " + e); - } - - // providers to validate what there is - // TODO: Validation should be done before persistence - AggregateConf instanceDescription = builder.getInstanceDescription(); - validateInstanceDefinition(sliderAM, instanceDescription, sliderFileSystem); - validateInstanceDefinition(provider, instanceDescription, sliderFileSystem); - } - - private void validateClientAndClusterResource(String clustername, - ConfTreeOperations clientResources) throws BadClusterStateException, - SliderException, IOException { - log.info("Validating upgrade resource definition with current cluster " - + "state (components and instance count)"); - Map clientComponentInstances = new HashMap<>(); - for (String componentName : clientResources.getComponentNames()) { - if (!SliderKeys.COMPONENT_AM.equals(componentName)) { - clientComponentInstances.put(componentName, clientResources - .getComponentOptInt(componentName, - COMPONENT_INSTANCES, -1)); - } - } - - AggregateConf clusterConf = null; - try { - clusterConf = loadPersistedClusterDescription(clustername); - } catch (LockAcquireFailedException e) { - log.warn("Failed to get a Lock on cluster resource : {}", e, e); - throw new BadClusterStateException( - "Failed to load client resource definition " + clustername + ": " + e, e); - } - Map clusterComponentInstances = new HashMap<>(); - for (Map.Entry> component : clusterConf - .getResources().components.entrySet()) { - if (!SliderKeys.COMPONENT_AM.equals(component.getKey())) { - clusterComponentInstances.put( - component.getKey(), - Integer.decode(component.getValue().get( - COMPONENT_INSTANCES))); - } - } - - // client and cluster should be an exact match - Iterator> clientComponentInstanceIt = clientComponentInstances - .entrySet().iterator(); - while (clientComponentInstanceIt.hasNext()) { - Map.Entry clientComponentInstanceEntry = clientComponentInstanceIt.next(); - if (clusterComponentInstances.containsKey(clientComponentInstanceEntry.getKey())) { - // compare instance count now and remove from both maps if they match - if (clusterComponentInstances - .get(clientComponentInstanceEntry.getKey()).intValue() == clientComponentInstanceEntry - .getValue().intValue()) { - clusterComponentInstances.remove(clientComponentInstanceEntry - .getKey()); - clientComponentInstanceIt.remove(); - } - } - } - - if (!clientComponentInstances.isEmpty() - || !clusterComponentInstances.isEmpty()) { - log.error("Mismatch found in upgrade resource definition and cluster " - + "resource state"); - if (!clientComponentInstances.isEmpty()) { - log.info("The upgrade resource definitions that do not match are:"); - for (Map.Entry clientComponentInstanceEntry : clientComponentInstances - .entrySet()) { - log.info(" Component Name: {}, Instance count: {}", - clientComponentInstanceEntry.getKey(), - clientComponentInstanceEntry.getValue()); - } - } - if (!clusterComponentInstances.isEmpty()) { - log.info("The cluster resources that do not match are:"); - for (Map.Entry clusterComponentInstanceEntry : clusterComponentInstances - .entrySet()) { - log.info(" Component Name: {}, Instance count: {}", - clusterComponentInstanceEntry.getKey(), - clusterComponentInstanceEntry.getValue()); - } - } - throw new BadConfigException("Resource definition provided for " - + "upgrade does not match with that of the currently running " - + "cluster.\nIf you are aware of what you are doing, rerun the " - + "command with " + Arguments.ARG_FORCE + " option."); - } - } - - protected void persistInstanceDefinition(boolean overwrite, - Path appconfdir, - InstanceBuilder builder) - throws IOException, SliderException, LockAcquireFailedException { - builder.persist(appconfdir, overwrite); - } - - @VisibleForTesting - public static void replaceTokens(ConfTree conf, - String userName, String clusterName) throws IOException { - Map newglobal = new HashMap<>(); - for (Entry entry : conf.global.entrySet()) { - newglobal.put(entry.getKey(), replaceTokens(entry.getValue(), - userName, clusterName)); - } - conf.global.putAll(newglobal); - - for (String component : conf.components.keySet()) { - Map newComponent = new HashMap<>(); - for (Entry entry : conf.components.get(component).entrySet()) { - newComponent.put(entry.getKey(), replaceTokens(entry.getValue(), - userName, clusterName)); - } - conf.components.get(component).putAll(newComponent); - } - - Map> newcred = new HashMap<>(); - for (Entry> entry : conf.credentials.entrySet()) { - List resultList = new ArrayList<>(); - for (String v : entry.getValue()) { - resultList.add(replaceTokens(v, userName, clusterName)); - } - newcred.put(replaceTokens(entry.getKey(), userName, clusterName), - resultList); - } - conf.credentials.clear(); - conf.credentials.putAll(newcred); - } - - private static String replaceTokens(String s, String userName, - String clusterName) throws IOException { - return s.replaceAll(Pattern.quote("${USER}"), userName) - .replaceAll(Pattern.quote("${USER_NAME}"), userName); - } - - public FsPermission getClusterDirectoryPermissions(Configuration conf) { - String clusterDirPermsOct = - conf.get(CLUSTER_DIRECTORY_PERMISSIONS, DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS); - return new FsPermission(clusterDirPermsOct); - } - - /** - * Verify that the Resource Manager is configured (on a non-HA cluster). - * with a useful error message - * @throws BadCommandArgumentsException the exception raised on an invalid config - */ - public void verifyBindingsDefined() throws BadCommandArgumentsException { - InetSocketAddress rmAddr = getRmAddress(getConfig()); - if (!getConfig().getBoolean(YarnConfiguration.RM_HA_ENABLED, false) - && !isAddressDefined(rmAddr)) { - throw new BadCommandArgumentsException( - E_NO_RESOURCE_MANAGER - + " in the argument " - + Arguments.ARG_MANAGER - + " or the configuration property " - + YarnConfiguration.RM_ADDRESS - + " value :" + rmAddr); - } - } - - /** - * Load and start a cluster specification. - * This assumes that all validation of args and cluster state - * have already taken place - * - * @param clustername name of the cluster. - * @param launchArgs launch arguments - * @param lifetime - * @return the exit code - * @throws YarnException - * @throws IOException - */ - protected int startCluster(String clustername, LaunchArgsAccessor launchArgs, - long lifetime) throws YarnException, IOException { - Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); - AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( - clustername, - clusterDirectory); - - LaunchedApplication launchedApplication = - launchApplication(clustername, clusterDirectory, instanceDefinition, - serviceArgs.isDebug(), lifetime); - - if (launchArgs.getOutputFile() != null) { - // output file has been requested. Get the app report and serialize it - ApplicationReport report = - launchedApplication.getApplicationReport(); - SerializedApplicationReport sar = new SerializedApplicationReport(report); - sar.submitTime = System.currentTimeMillis(); - ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser(); - serDeser.save(sar, launchArgs.getOutputFile()); - } - int waittime = launchArgs.getWaittime(); - if (waittime > 0) { - return waitForAppRunning(launchedApplication, waittime, waittime); - } else { - // no waiting - return EXIT_SUCCESS; - } + return newTimeout; } /** @@ -1968,415 +1588,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } } - /** - * Load the instance definition. - * @param name cluster name - * @param resolved flag to indicate the cluster should be resolved - * @return the loaded configuration - * @throws IOException IO problems - * @throws SliderException slider explicit issues - * @throws UnknownApplicationInstanceException if the file is not found - */ - public AggregateConf loadInstanceDefinition(String name, - boolean resolved) throws - IOException, - SliderException { - - Path clusterDirectory = sliderFileSystem.buildClusterDirPath(name); - AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( - name, - clusterDirectory); - if (resolved) { - instanceDefinition.resolve(); - } - return instanceDefinition; - - } - - protected AppMasterLauncher setupAppMasterLauncher(String clustername, - Path clusterDirectory, AggregateConf instanceDefinition, boolean debugAM, - long lifetime) - throws YarnException, IOException{ - deployedClusterName = clustername; - validateClusterName(clustername); - verifyNoLiveClusters(clustername, "Launch"); - Configuration config = getConfig(); - lookupZKQuorum(); - boolean clusterSecure = isHadoopClusterSecure(config); - //create the Slider AM provider -this helps set up the AM - SliderAMClientProvider sliderAM = new SliderAMClientProvider(config); - - instanceDefinition.resolve(); - launchedInstanceDefinition = instanceDefinition; - - ConfTreeOperations internalOperations = instanceDefinition.getInternalOperations(); - MapOperations internalOptions = internalOperations.getGlobalOptions(); - ConfTreeOperations resourceOperations = instanceDefinition.getResourceOperations(); - ConfTreeOperations appOperations = instanceDefinition.getAppConfOperations(); - Path generatedConfDirPath = - createPathThatMustExist(internalOptions.getMandatoryOption( - INTERNAL_GENERATED_CONF_PATH)); - Path snapshotConfPath = - createPathThatMustExist(internalOptions.getMandatoryOption( - INTERNAL_SNAPSHOT_CONF_PATH)); - - - // cluster Provider - AbstractClientProvider provider = createClientProvider( - internalOptions.getMandatoryOption(INTERNAL_PROVIDER_NAME)); - if (log.isDebugEnabled()) { - log.debug(instanceDefinition.toString()); - } - MapOperations sliderAMResourceComponent = - resourceOperations.getOrAddComponent(SliderKeys.COMPONENT_AM); - MapOperations resourceGlobalOptions = resourceOperations.getGlobalOptions(); - - // add the tags if available - Set applicationTags = provider.getApplicationTags(sliderFileSystem, - appOperations, clustername); - - Credentials credentials = null; - if (clusterSecure) { - // pick up oozie credentials - credentials = CredentialUtils.loadTokensFromEnvironment(System.getenv(), - config); - if (credentials == null) { - // nothing from oozie, so build up directly - credentials = new Credentials( - UserGroupInformation.getCurrentUser().getCredentials()); - CredentialUtils.addRMRenewableFSDelegationTokens(config, - sliderFileSystem.getFileSystem(), - credentials); - CredentialUtils.addRMDelegationToken(yarnClient, credentials); - - } else { - log.info("Using externally supplied credentials to launch AM"); - } - } - - AppMasterLauncher amLauncher = new AppMasterLauncher(clustername, - SliderKeys.APP_TYPE, - config, - sliderFileSystem, - yarnClient, - clusterSecure, - sliderAMResourceComponent, - resourceGlobalOptions, - applicationTags, - credentials); - - ApplicationId appId = amLauncher.getApplicationId(); - // set the application name; - amLauncher.setKeepContainersOverRestarts(true); - // set lifetime in submission context; - Map appTimeout = new HashMap<>(); - if (lifetime > 0) { - appTimeout.put(ApplicationTimeoutType.LIFETIME, lifetime); - } - amLauncher.submissionContext.setApplicationTimeouts(appTimeout); - int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0); - amLauncher.setMaxAppAttempts(maxAppAttempts); - - sliderFileSystem.purgeAppInstanceTempFiles(clustername); - Path tempPath = sliderFileSystem.createAppInstanceTempPath( - clustername, - appId.toString() + "/am"); - String libdir = "lib"; - Path libPath = new Path(tempPath, libdir); - sliderFileSystem.getFileSystem().mkdirs(libPath); - log.debug("FS={}, tempPath={}, libdir={}", sliderFileSystem, tempPath, libPath); - - // set local resources for the application master - // local files or archives as needed - // In this scenario, the jar file for the application master is part of the local resources - Map localResources = amLauncher.getLocalResources(); - - // look for the configuration directory named on the command line - boolean hasServerLog4jProperties = false; - Path remoteConfPath = null; - String relativeConfDir = null; - String confdirProp = System.getProperty(SliderKeys.PROPERTY_CONF_DIR); - if (isUnset(confdirProp)) { - log.debug("No local configuration directory provided as system property"); - } else { - File confDir = new File(confdirProp); - if (!confDir.exists()) { - throw new BadConfigException(E_CONFIGURATION_DIRECTORY_NOT_FOUND, - confDir); - } - Path localConfDirPath = createLocalPath(confDir); - remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR); - log.debug("Slider configuration directory is {}; remote to be {}", - localConfDirPath, remoteConfPath); - copyDirectory(config, localConfDirPath, remoteConfPath, null); - - File log4jserver = - new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME); - hasServerLog4jProperties = log4jserver.isFile(); - } - if (!hasServerLog4jProperties) { - // check for log4j properties in hadoop conf dir - String hadoopConfDir = System.getenv(ApplicationConstants.Environment - .HADOOP_CONF_DIR.name()); - if (hadoopConfDir != null) { - File localFile = new File(hadoopConfDir, SliderKeys - .LOG4J_SERVER_PROP_FILENAME); - if (localFile.exists()) { - Path localFilePath = createLocalPath(localFile); - remoteConfPath = new Path(clusterDirectory, - SliderKeys.SUBMITTED_CONF_DIR); - Path remoteFilePath = new Path(remoteConfPath, SliderKeys - .LOG4J_SERVER_PROP_FILENAME); - copy(config, localFilePath, remoteFilePath); - hasServerLog4jProperties = true; - } - } - } - // the assumption here is that minimr cluster => this is a test run - // and the classpath can look after itself - - boolean usingMiniMRCluster = getUsingMiniMRCluster(); - if (!usingMiniMRCluster) { - - log.debug("Destination is not a MiniYARNCluster -copying full classpath"); - - // insert conf dir first - if (remoteConfPath != null) { - relativeConfDir = SliderKeys.SUBMITTED_CONF_DIR; - Map submittedConfDir = - sliderFileSystem.submitDirectory(remoteConfPath, - relativeConfDir); - mergeMaps(localResources, submittedConfDir); - } - } - // build up the configuration - // IMPORTANT: it is only after this call that site configurations - // will be valid. - - propagatePrincipals(config, instanceDefinition); - // validate security data - -/* - // turned off until tested - SecurityConfiguration securityConfiguration = - new SecurityConfiguration(config, - instanceDefinition, clustername); - -*/ - Configuration clientConfExtras = new Configuration(false); - // then build up the generated path. - FsPermission clusterPerms = getClusterDirectoryPermissions(config); - copyDirectory(config, snapshotConfPath, generatedConfDirPath, - clusterPerms); - - - // standard AM resources - sliderAM.prepareAMAndConfigForLaunch(sliderFileSystem, - config, - amLauncher, - instanceDefinition, - snapshotConfPath, - generatedConfDirPath, - clientConfExtras, - libdir, - tempPath, - usingMiniMRCluster); - //add provider-specific resources - provider.prepareAMAndConfigForLaunch(sliderFileSystem, - config, - amLauncher, - instanceDefinition, - snapshotConfPath, - generatedConfDirPath, - clientConfExtras, - libdir, - tempPath, - usingMiniMRCluster); - - // now that the site config is fully generated, the provider gets - // to do a quick review of them. - log.debug("Preflight validation of cluster configuration"); - - - sliderAM.preflightValidateClusterConfiguration(sliderFileSystem, - clustername, - config, - instanceDefinition, - clusterDirectory, - generatedConfDirPath, - clusterSecure - ); - - provider.preflightValidateClusterConfiguration(sliderFileSystem, - clustername, - config, - instanceDefinition, - clusterDirectory, - generatedConfDirPath, - clusterSecure - ); - - - if (!(provider instanceof DockerClientProvider)) { - Path imagePath = - extractImagePath(sliderFileSystem, internalOptions); - if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) { - log.debug("Registered image path {}", imagePath); - } - } - - // build the environment - amLauncher.putEnv( - buildEnvMap(sliderAMResourceComponent)); - ClasspathConstructor classpath = buildClasspath(relativeConfDir, - libdir, - getConfig(), - sliderFileSystem, - usingMiniMRCluster); - amLauncher.setClasspath(classpath); - //add english env - amLauncher.setEnv("LANG", "en_US.UTF-8"); - amLauncher.setEnv("LC_ALL", "en_US.UTF-8"); - amLauncher.setEnv("LANGUAGE", "en_US.UTF-8"); - amLauncher.maybeSetEnv(HADOOP_JAAS_DEBUG, - System.getenv(HADOOP_JAAS_DEBUG)); - amLauncher.putEnv(getAmLaunchEnv(config)); - - for (Map.Entry envs : getSystemEnv().entrySet()) { - log.debug("System env {}={}", envs.getKey(), envs.getValue()); - } - if (log.isDebugEnabled()) { - log.debug("AM classpath={}", classpath); - log.debug("Environment Map:\n{}", - stringifyMap(amLauncher.getEnv())); - log.debug("Files in lib path\n{}", sliderFileSystem.listFSDir(libPath)); - } - - // rm address - - InetSocketAddress rmSchedulerAddress; - try { - rmSchedulerAddress = getRmSchedulerAddress(config); - } catch (IllegalArgumentException e) { - throw new BadConfigException("%s Address invalid: %s", - YarnConfiguration.RM_SCHEDULER_ADDRESS, - config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS)); - } - String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress); - - JavaCommandLineBuilder commandLine = new JavaCommandLineBuilder(); - // insert any JVM options); - sliderAM.addJVMOptions(instanceDefinition, commandLine); - // enable asserts - commandLine.enableJavaAssertions(); - - // if the conf dir has a slideram-log4j.properties, switch to that - if (hasServerLog4jProperties) { - commandLine.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME); - commandLine.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR); - } - - // add the AM sevice entry point - commandLine.add(SliderAppMaster.SERVICE_CLASSNAME); - - // create action and the cluster name - commandLine.add(ACTION_CREATE, clustername); - - // debug - if (debugAM) { - commandLine.add(Arguments.ARG_DEBUG); - } - - // set the cluster directory path - commandLine.add(Arguments.ARG_CLUSTER_URI, clusterDirectory.toUri()); - - if (!isUnset(rmAddr)) { - commandLine.add(Arguments.ARG_RM_ADDR, rmAddr); - } - - if (serviceArgs.getFilesystemBinding() != null) { - commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemBinding()); - } - - // pass the registry binding - commandLine.addConfOptionToCLI(config, RegistryConstants.KEY_REGISTRY_ZK_ROOT, - RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT); - commandLine.addMandatoryConfOption(config, RegistryConstants.KEY_REGISTRY_ZK_QUORUM); - - if (clusterSecure) { - // if the cluster is secure, make sure that - // the relevant security settings go over - commandLine.addConfOption(config, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); - } - - // copy over any/all YARN RM client values, in case the server-side XML conf file - // has the 0.0.0.0 address - commandLine.addConfOptions(config, - YarnConfiguration.RM_ADDRESS, - YarnConfiguration.RM_CLUSTER_ID, - YarnConfiguration.RM_HOSTNAME, - YarnConfiguration.RM_PRINCIPAL); - - // write out the path output - commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM); - - String cmdStr = commandLine.build(); - log.debug("Completed setting up app master command {}", cmdStr); - - amLauncher.addCommandLine(commandLine); - - // the Slider AM gets to configure the AM requirements, not the custom provider - sliderAM.prepareAMResourceRequirements(sliderAMResourceComponent, - amLauncher.getResource()); - - - // Set the priority for the application master - amLauncher.setPriority(config.getInt(KEY_YARN_QUEUE_PRIORITY, - DEFAULT_YARN_QUEUE_PRIORITY)); - - // Set the queue to which this application is to be submitted in the RM - // Queue for App master - String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE); - String suppliedQueue = internalOperations.getGlobalOptions().get(INTERNAL_QUEUE); - if(!isUnset(suppliedQueue)) { - amQueue = suppliedQueue; - log.info("Using queue {} for the application instance.", amQueue); - } - - if (isSet(amQueue)) { - amLauncher.setQueue(amQueue); - } - return amLauncher; - } - - /** - * - * @param clustername name of the cluster - * @param clusterDirectory cluster dir - * @param instanceDefinition the instance definition - * @param debugAM enable debug AM options - * @param lifetime - * @return the launched application - * @throws YarnException - * @throws IOException - */ - public LaunchedApplication launchApplication(String clustername, Path clusterDirectory, - AggregateConf instanceDefinition, boolean debugAM, long lifetime) - throws YarnException, IOException { - - AppMasterLauncher amLauncher = setupAppMasterLauncher(clustername, - clusterDirectory, - instanceDefinition, - debugAM, lifetime); - - applicationId = amLauncher.getApplicationId(); - log.info("Submitting application {}", applicationId); - - // submit the application - LaunchedApplication launchedApplication = amLauncher.submitApplication(); - return launchedApplication; - } - protected Map getAmLaunchEnv(Configuration config) { String sliderAmLaunchEnv = config.get(KEY_AM_LAUNCH_ENV); log.debug("{} = {}", KEY_AM_LAUNCH_ENV, sliderAmLaunchEnv); @@ -2431,95 +1642,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return placeholderKeyValueMap; } - private void propagatePythonExecutable(Configuration config, - AggregateConf instanceDefinition) { - String pythonExec = config.get( - PYTHON_EXECUTABLE_PATH); - if (pythonExec != null) { - instanceDefinition.getAppConfOperations().getGlobalOptions().putIfUnset( - PYTHON_EXECUTABLE_PATH, - pythonExec); - } - } - - - /** - * Wait for the launched app to be accepted in the time - * and, optionally running. - *

- * If the application - * - * @param launchedApplication application - * @param acceptWaitMillis time in millis to wait for accept - * @param runWaitMillis time in millis to wait for the app to be running. - * May be null, in which case no wait takes place - * @return exit code: success - * @throws YarnException - * @throws IOException - */ - public int waitForAppRunning(LaunchedApplication launchedApplication, - int acceptWaitMillis, int runWaitMillis) throws YarnException, IOException { - assert launchedApplication != null; - int exitCode; - // wait for the submit state to be reached - ApplicationReport report = launchedApplication.monitorAppToState( - YarnApplicationState.ACCEPTED, - new Duration(acceptWaitMillis)); - - // may have failed, so check that - if (hasAppFinished(report)) { - exitCode = buildExitCode(report); - } else { - // exit unless there is a wait - - - if (runWaitMillis != 0) { - // waiting for state to change - Duration duration = new Duration(runWaitMillis * 1000); - duration.start(); - report = launchedApplication.monitorAppToState( - YarnApplicationState.RUNNING, duration); - if (report != null && - report.getYarnApplicationState() == YarnApplicationState.RUNNING) { - exitCode = EXIT_SUCCESS; - } else { - exitCode = buildExitCode(report); - } - } else { - exitCode = EXIT_SUCCESS; - } - } - return exitCode; - } - - - /** - * Propagate any critical principals from the current site config down to the HBase one. - * @param config config to read from - * @param clusterSpec cluster spec - */ - private void propagatePrincipals(Configuration config, - AggregateConf clusterSpec) { - String dfsPrincipal = config.get(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); - if (dfsPrincipal != null) { - String siteDfsPrincipal = SITE_XML_PREFIX + DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; - clusterSpec.getAppConfOperations().getGlobalOptions().putIfUnset( - siteDfsPrincipal, - dfsPrincipal); - } - } - - /** - * Create a path that must exist in the cluster fs - * @param uri uri to create - * @return the path - * @throws FileNotFoundException if the path does not exist - */ - public Path createPathThatMustExist(String uri) throws - SliderException, IOException { - return sliderFileSystem.createPathThatMustExist(uri); - } - /** * verify that a live cluster isn't there * @param clustername cluster name @@ -2527,7 +1649,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe * @throws SliderException with exit code EXIT_CLUSTER_LIVE * if a cluster of that name is either live or starting up. */ - public void verifyNoLiveClusters(String clustername, String action) throws + public void verifyNoLiveApp(String clustername, String action) throws IOException, YarnException { List existing = findAllLiveInstances(clustername); @@ -2554,11 +1676,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return deployedClusterName; } - @VisibleForTesting - public void setDeployedClusterName(String deployedClusterName) { - this.deployedClusterName = deployedClusterName; - } - /** * ask if the client is using a mini MR cluster * @return true if they are @@ -2568,109 +1685,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe false); } - /** - * Get the application name used in the zookeeper root paths - * @return an application-specific path in ZK - */ - private String getAppName() { - return "slider"; - } - - /** - * Wait for the app to start running (or go past that state) - * @param duration time to wait - * @return the app report; null if the duration turned out - * @throws YarnException YARN or app issues - * @throws IOException IO problems - */ - @VisibleForTesting - public ApplicationReport monitorAppToRunning(Duration duration) - throws YarnException, IOException { - return monitorAppToState(YarnApplicationState.RUNNING, duration); - } - - /** - * Build an exit code for an application from its report. - * If the report parameter is null, its interpreted as a timeout - * @param report report application report - * @return the exit code - * @throws IOException - * @throws YarnException - */ - private int buildExitCode(ApplicationReport report) throws - IOException, - YarnException { - if (null == report) { - return EXIT_TIMED_OUT; - } - - YarnApplicationState state = report.getYarnApplicationState(); - FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); - switch (state) { - case FINISHED: - if (FinalApplicationStatus.SUCCEEDED == dsStatus) { - log.info("Application has completed successfully"); - return EXIT_SUCCESS; - } else { - log.info("Application finished unsuccessfully." + - "YarnState = {}, DSFinalStatus = {} Breaking monitoring loop", - state, dsStatus); - return EXIT_YARN_SERVICE_FINISHED_WITH_ERROR; - } - - case KILLED: - log.info("Application did not finish. YarnState={}, DSFinalStatus={}", - state, dsStatus); - return EXIT_YARN_SERVICE_KILLED; - - case FAILED: - log.info("Application Failed. YarnState={}, DSFinalStatus={}", state, - dsStatus); - return EXIT_YARN_SERVICE_FAILED; - - default: - //not in any of these states - return EXIT_SUCCESS; - } - } - - /** - * Monitor the submitted application for reaching the requested state. - * Will also report if the app reaches a later state (failed, killed, etc) - * Kill application if duration!= null & time expires. - * Prerequisite: the applicatin was launched. - * @param desiredState desired state. - * @param duration how long to wait -must be more than 0 - * @return the application report -null on a timeout - * @throws YarnException - * @throws IOException - */ - @VisibleForTesting - public ApplicationReport monitorAppToState( - YarnApplicationState desiredState, - Duration duration) - throws YarnException, IOException { - LaunchedApplication launchedApplication = - new LaunchedApplication(applicationId, yarnClient); - return launchedApplication.monitorAppToState(desiredState, duration); - } - - @Override - public ApplicationReport getApplicationReport() throws - IOException, - YarnException { - return getApplicationReport(applicationId); - } - - @Override - public boolean forceKillApplication(String reason) - throws YarnException, IOException { - if (applicationId != null) { - new LaunchedApplication(applicationId, yarnClient).forceKill(reason); - return true; - } - return false; - } /** * List Slider instances belonging to a specific user with a specific app @@ -2720,23 +1734,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } } - /** - * Retrieve a list of all live instances. If clustername is supplied then it - * returns this specific cluster, if and only if it exists and is live. - * - * @param clustername - * cluster name (if looking for a specific live cluster) - * @return the list of application names which satisfies the list criteria - * @throws IOException - * @throws YarnException - */ - public Set getApplicationList(String clustername) - throws IOException, YarnException { - ActionListArgs args = new ActionListArgs(); - args.live = true; - return getApplicationList(clustername, args); - } - /** * Retrieve a list of application instances satisfying the query criteria. * @@ -2757,8 +1754,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // the above call throws an exception so the return is not really required return Collections.emptySet(); } - verifyBindingsDefined(); - boolean live = args.live; String state = args.state; boolean listContainers = args.containers; @@ -2868,29 +1863,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } } - /** - * Enumerate slider instances for the current user, and the - * most recent app report, where available. - * @param listOnlyInState boolean to indicate that the instances should - * only include those in a YARN state - * minAppState <= currentState <= maxAppState - * - * @param minAppState minimum application state to include in enumeration. - * @param maxAppState maximum application state to include - * @return a map of application instance name to description - * @throws IOException Any IO problem - * @throws YarnException YARN problems - */ - @Override - public Map enumSliderInstances( - boolean listOnlyInState, - YarnApplicationState minAppState, - YarnApplicationState maxAppState) - throws IOException, YarnException { - return yarnAppListClient.enumSliderInstances(listOnlyInState, - minAppState, - maxAppState); - } /** * Extract the state of a Yarn application --state argument @@ -2928,22 +1900,16 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe @Override @VisibleForTesting - public int actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException { - validateClusterName(name); - Map roleMap = args.getComponentMap(); - // throw usage exception if no changes proposed - if (roleMap.size() == 0) { - actionHelp(ACTION_FLEX); + public void actionFlex(String appName, ActionFlexArgs args) + throws YarnException, IOException { + Component component = new Component(); + component.setNumberOfContainers(args.getNumberOfContainers()); + if (StringUtils.isEmpty(args.getComponent())) { + component.setName("DEFAULT"); + } else { + component.setName(args.getComponent()); } - verifyBindingsDefined(); - log.debug("actionFlex({})", name); - Map roleInstances = new HashMap<>(); - for (Map.Entry roleEntry : roleMap.entrySet()) { - String key = roleEntry.getKey(); - String val = roleEntry.getValue(); - roleInstances.put(key, val); - } - return flex(name, roleInstances); + flex(appName, component); } @Override @@ -2954,7 +1920,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } public int actionExists(String name, ActionExistsArgs args) throws YarnException, IOException { - verifyBindingsDefined(); validateClusterName(name); boolean checkLive = args.live; log.debug("actionExists({}, {}, {})", name, checkLive, args.state); @@ -3049,14 +2014,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return clusterOps.echo(message); } - /** - * Get at the service registry operations - * @return registry client -valid after the service is inited. - */ - public YarnAppListClient getYarnAppListClient() { - return yarnAppListClient; - } - /** * Find an instance of an application belonging to the current user. * @param appname application name @@ -3128,20 +2085,20 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return EXIT_SUCCESS; } - ClusterDescription status = verifyAndGetClusterDescription(clustername); + Application application = getApplication(clustername); String outfile = statusArgs.getOutput(); if (outfile == null) { - log.info(status.toJsonString()); + log.info(application.toString()); } else { - status.save(new File(outfile).getAbsoluteFile()); + jsonSerDeser.save(application, new File(statusArgs.getOutput())); } return EXIT_SUCCESS; } @Override - public String actionStatus(String clustername) + public Application actionStatus(String clustername) throws YarnException, IOException { - return verifyAndGetClusterDescription(clustername).toJsonString(); + return getApplication(clustername); } private void queryAndPrintLifetime(String appName) @@ -3170,13 +2127,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } } - private ClusterDescription verifyAndGetClusterDescription(String clustername) - throws YarnException, IOException { - verifyBindingsDefined(); - validateClusterName(clustername); - return getClusterDescription(clustername); - } - @Override public int actionVersion() { SliderVersionInfo.loadAndPrintVersionInfo(log); @@ -3184,269 +2134,106 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } @Override - public int actionFreeze(String clustername, - ActionFreezeArgs freezeArgs) throws YarnException, IOException { - verifyBindingsDefined(); - validateClusterName(clustername); - int waittime = freezeArgs.getWaittime(); - String text = freezeArgs.message; - boolean forcekill = freezeArgs.force; - log.debug("actionFreeze({}, reason={}, wait={}, force={})", clustername, - text, - waittime, - forcekill); - - //is this actually a known cluster? - sliderFileSystem.locateInstanceDefinition(clustername); - ApplicationReport app = findInstance(clustername); + public void actionStop(String appName, ActionFreezeArgs freezeArgs) + throws YarnException, IOException { + validateClusterName(appName); + ApplicationReport app = findInstance(appName); if (app == null) { - // exit early - log.info("Cluster {} not running", clustername); - // not an error to stop a stopped cluster - return EXIT_SUCCESS; - } - log.debug("App to stop was found: {}:\n{}", clustername, - new OnDemandReportStringifier(app)); - if (app.getYarnApplicationState().ordinal() >= - YarnApplicationState.FINISHED.ordinal()) { - log.info("Cluster {} is in a terminated state {}", clustername, - app.getYarnApplicationState()); - return EXIT_SUCCESS; + throw new ApplicationNotFoundException( + "Application " + appName + " doesn't exist in RM."); } - // IPC request for a managed shutdown is only possible if the app is running. - // so we need to force kill if the app is accepted or submitted - if (!forcekill - && app.getYarnApplicationState().ordinal() < YarnApplicationState.RUNNING.ordinal()) { - log.info("Cluster {} is in a pre-running state {}. Force killing it", clustername, + if (app.getYarnApplicationState().ordinal() >= YarnApplicationState.FINISHED + .ordinal()) { + log.info("Application {} is in a terminated state {}", appName, app.getYarnApplicationState()); - forcekill = true; + return; } - LaunchedApplication application = new LaunchedApplication(yarnClient, app); - applicationId = application.getApplicationId(); - - if (forcekill) { - // escalating to forced kill - application.kill("Forced stop of " + clustername + ": " + text); - } else { - try { - SliderClusterProtocol appMaster = connect(app); - Messages.StopClusterRequestProto r = - Messages.StopClusterRequestProto - .newBuilder() - .setMessage(text) - .build(); - appMaster.stopCluster(r); - - log.debug("Cluster stop command issued"); - - } catch (YarnException e) { - log.warn("Exception while trying to terminate {}", clustername, e); - return EXIT_FALSE; - } catch (IOException e) { - log.warn("Exception while trying to terminate {}", clustername, e); - return EXIT_FALSE; - } - } - - //wait for completion. We don't currently return an exception during this process - //as the stop operation has been issued, this is just YARN. try { - if (waittime > 0) { - ApplicationReport applicationReport = - application.monitorAppToState(YarnApplicationState.FINISHED, - new Duration(waittime * 1000)); - if (applicationReport == null) { - log.info("application did not shut down in time"); - return EXIT_FALSE; - } - } - - } catch (YarnException | IOException e) { - log.warn("Exception while waiting for the application {} to shut down: {}", - clustername, e); + SliderClusterProtocol appMaster = connect(app); + Messages.StopClusterRequestProto r = + Messages.StopClusterRequestProto.newBuilder() + .setMessage(freezeArgs.message).build(); + appMaster.stopCluster(r); + log.info("Application " + appName + " is gracefully stopped."); + } catch (IOException | YarnException e){ + log.info("Failed to stop " + appName + + " gracefully, forcefully kill the app."); + yarnClient.killApplication(app.getApplicationId(), freezeArgs.message); } - - return EXIT_SUCCESS; } @Override - public int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException { - validateClusterName(clustername); - verifyBindingsDefined(); - // see if it is actually running and bail out; - verifyNoLiveClusters(clustername, "Start"); - - //start the cluster - return startCluster(clustername, thaw, thaw.lifetime); - } - - /** - * Implement flexing - * @param clustername name of the cluster - * @param roleInstances map of new role instances - * @return EXIT_SUCCESS if the #of nodes in a live cluster changed - * @throws YarnException - * @throws IOException - */ - public int flex(String clustername, Map roleInstances) + public int actionStart(String appName, ActionThawArgs thaw) throws YarnException, IOException { - verifyBindingsDefined(); - validateClusterName(clustername); - Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); - AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved( - clustername, - clusterDirectory); + validateClusterName(appName); + // see if it is actually running and bail out; + verifyNoLiveApp(appName, "Thaw"); + Path appDir = sliderFileSystem.buildClusterDirPath(appName); + Path appJson = new Path(appDir, appName + ".json"); + Application application = + jsonSerDeser.load(sliderFileSystem.getFileSystem(), appJson); + submitApp(application); + return 0; + } - ConfTreeOperations resources = - instanceDefinition.getResourceOperations(); - for (Map.Entry entry : roleInstances.entrySet()) { - String role = entry.getKey(); - String updateCountStr = entry.getValue(); - int currentCount = 0; - MapOperations component = resources.getOrAddComponent(role); - try { - // check if a relative count is specified - if (updateCountStr.startsWith("+") || updateCountStr.startsWith("-")) { - int updateCount = Integer.parseInt(updateCountStr); - // if component was specified before, get the current count - if (component.get(COMPONENT_INSTANCES) != null) { - currentCount = Integer.parseInt(component.get(COMPONENT_INSTANCES)); - if (currentCount + updateCount < 0) { - throw new BadCommandArgumentsException("The requested count " + - "of \"%s\" for role %s makes the total number of " + - "instances negative: \"%s\"", updateCount, role, - currentCount+updateCount); - } - else { - component.put(COMPONENT_INSTANCES, - Integer.toString(currentCount+updateCount)); - } - } - else { - if (updateCount < 0) { - throw new BadCommandArgumentsException("Invalid to request " + - "negative count of \"%s\" for role %s", updateCount, role); - } - else { - Map map = new HashMap<>(); - resources.confTree.components.put(role, map); - component = new MapOperations(role, map); - component.put(COMPONENT_INSTANCES, Integer.toString(updateCount)); - } - } - } - else { - int count = Integer.parseInt(updateCountStr); - resources.getOrAddComponent(role).put(COMPONENT_INSTANCES, - Integer.toString(count)); - } + public long flex(String appName, Component component) + throws YarnException, IOException { + validateClusterName(appName); + Path appDir = sliderFileSystem.buildClusterDirPath(appName); + Path appJson = new Path(appDir, appName + ".json"); + Application persistedApp = + jsonSerDeser.load(sliderFileSystem.getFileSystem(), appJson); + long original = 0; + boolean foundComponent = false; + for (Component persistedComp : persistedApp.getComponents()) { + if (persistedComp.getName().equals(component.getName())) { + original = persistedComp.getNumberOfContainers(); + persistedComp.setNumberOfContainers(component.getNumberOfContainers()); + foundComponent = true; + break; } - catch (NumberFormatException e) { - throw new BadCommandArgumentsException("Requested count of role %s" + - " is not a number: \"%s\"", - role, updateCountStr); - } - - log.debug("Flexed cluster specification ( {} -> {}) : \n{}", - role, - updateCountStr, - resources); } - SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig()); - AbstractClientProvider provider = createClientProvider( - instanceDefinition.getInternalOperations().getGlobalOptions().getMandatoryOption( - INTERNAL_PROVIDER_NAME)); - // slider provider to validate what there is - validateInstanceDefinition(sliderAM, instanceDefinition, sliderFileSystem); - validateInstanceDefinition(provider, instanceDefinition, sliderFileSystem); - - int exitCode = EXIT_FALSE; - // save the specification - try { - InstanceIO.saveInstanceDefinition(sliderFileSystem, clusterDirectory, - instanceDefinition); - } catch (LockAcquireFailedException e) { - // lock failure - log.debug("Failed to lock dir {}", clusterDirectory, e); - log.warn("Failed to save new resource definition to {} : {}", clusterDirectory, e); + if (!foundComponent) { + throw new YarnException("Component " + component.getName() + + " does not exist in app definition."); } + jsonSerDeser + .save(sliderFileSystem.getFileSystem(), appJson, persistedApp, true); + log.info("Updated app definition file for component " + component); - // now see if it is actually running and tell it about the update if it is - ApplicationReport instance = findInstance(clustername); + ApplicationReport instance = findInstance(appName); if (instance != null) { - log.info("Flexing running cluster"); + log.info("Flexing running app " + appName); SliderClusterProtocol appMaster = connect(instance); - SliderClusterOperations clusterOps = new SliderClusterOperations(appMaster); - clusterOps.flex(instanceDefinition.getResources()); - log.info("application instance size updated"); - exitCode = EXIT_SUCCESS; + SliderClusterOperations clusterOps = + new SliderClusterOperations(appMaster); + clusterOps.flex(component); + log.info( + "Application name = " + appName + ", Component name = " + component + .getName() + ", number of containers updated from " + original + + " to " + component.getNumberOfContainers()); } else { - log.info("No running instance to update"); + String message = "Application " + appName + "does not exist in RM. "; + throw new YarnException(message); } - return exitCode; + return original; } /** - * Validate an instance definition against a provider. - * @param provider the provider performing the validation - * @param instanceDefinition the instance definition - * @throws SliderException if invalid. - */ - protected void validateInstanceDefinition(AbstractClientProvider provider, - AggregateConf instanceDefinition, SliderFileSystem fs) throws SliderException { - try { - provider.validateInstanceDefinition(instanceDefinition, fs); - } catch (SliderException e) { - //problem, reject it - log.info("Error {} validating application instance definition ", e.getMessage()); - log.debug("Error validating application instance definition ", e); - log.info(instanceDefinition.toString()); - throw e; - } - } - - - /** - * Load the persistent cluster description - * @param clustername name of the cluster - * @return the description in the filesystem - * @throws IOException any problems loading -including a missing file - */ - @VisibleForTesting - public AggregateConf loadPersistedClusterDescription(String clustername) - throws IOException, SliderException, LockAcquireFailedException { - Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername); - ConfPersister persister = new ConfPersister(sliderFileSystem, clusterDirectory); - AggregateConf instanceDescription = new AggregateConf(); - persister.load(instanceDescription); - return instanceDescription; - } - - /** - * Connect to a live cluster and get its current state - * @param clustername the cluster name - * @return its description - */ - @VisibleForTesting - public ClusterDescription getClusterDescription(String clustername) throws - YarnException, - IOException { - SliderClusterOperations clusterOperations = - createClusterOperations(clustername); - return clusterOperations.getClusterDescription(); - } - - /** - * Connect to the cluster and get its current state + * Connect to a live cluster and get its current state + * + * @param appName the cluster name * @return its description */ @VisibleForTesting - public ClusterDescription getClusterDescription() throws - YarnException, - IOException { - return getClusterDescription(getDeployedClusterName()); + public Application getApplication(String appName) + throws YarnException, IOException { + validateClusterName(appName); + SliderClusterOperations clusterOperations = + createClusterOperations(appName); + return clusterOperations.getApplication(); } /** @@ -3498,14 +2285,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return createClusterOperations().listClusterNodes(uuids); } - /** - * Get the instance definition from the far end - */ - @VisibleForTesting - public AggregateConf getLiveInstanceDefinition() throws IOException, YarnException { - return createClusterOperations().getInstanceDefinition(); - } - /** * Bond to a running cluster * @param clustername cluster name @@ -3515,7 +2294,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe private SliderClusterProtocol bondToCluster(String clustername) throws YarnException, IOException { - verifyBindingsDefined(); if (clustername == null) { throw unknownClusterException("(undefined)"); } @@ -3601,23 +2379,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return yarnClient.getApplications(); } - @VisibleForTesting - public ApplicationReport getApplicationReport(ApplicationId appId) - throws YarnException, IOException { - return new LaunchedApplication(appId, yarnClient).getApplicationReport(); - } - - /** - * The configuration used for deployment (after resolution). - * Non-null only after the client has launched the application - * @return the resolved configuration or null - */ - @VisibleForTesting - public AggregateConf getLaunchedInstanceDefinition() { - return launchedInstanceDefinition; - } - - @Override public int actionResolve(ActionResolveArgs args) throws YarnException, IOException { @@ -3793,7 +2554,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } else if (diagnosticArgs.all) { actionDiagnosticAll(diagnosticArgs); } else if (diagnosticArgs.level) { - actionDiagnosticIntelligent(diagnosticArgs); + // agent is removed } else { // it's an unknown option log.info(CommonArgs.usage(serviceArgs, ACTION_DIAGNOSTICS)); @@ -3806,70 +2567,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return EXIT_SUCCESS; } - private void actionDiagnosticIntelligent(ActionDiagnosticArgs diagnosticArgs) - throws YarnException, IOException, URISyntaxException { - // not using member variable clustername because we want to place - // application name after --application option and member variable - // cluster name has to be put behind action - String clusterName = diagnosticArgs.name; - requireArgumentSet(Arguments.ARG_NAME, clusterName); - - try { - validateClientConfigFile(); - log.info("Slider-client.xml is accessible"); - } catch (IOException e) { - // we are catching exceptions here because those are indication of - // validation result, and we need to print them here - log.error("validation of slider-client.xml fails because: " + e, e); - return; - } - SliderClusterOperations clusterOperations = createClusterOperations(clusterName); - // cluster not found exceptions will be thrown upstream - ClusterDescription clusterDescription = clusterOperations - .getClusterDescription(); - log.info("Slider AppMaster is accessible"); - - if (clusterDescription.state == StateValues.STATE_LIVE) { - AggregateConf instanceDefinition = clusterOperations - .getInstanceDefinition(); - String imagePath = instanceDefinition.getInternalOperations().get( - INTERNAL_APPLICATION_IMAGE_PATH); - // if null, that means slider uploaded the agent tarball for the user - // and we need to use where slider has put - if (imagePath == null) { - ApplicationReport appReport = findInstance(clusterName); - Path path1 = sliderFileSystem.getTempPathForCluster(clusterName); - if (appReport != null) { - Path subPath = new Path(path1, appReport.getApplicationId() - .toString() + "/agent"); - imagePath = subPath.toString(); - String pathStr = imagePath + "/" + AgentKeys.AGENT_TAR; - try { - validateHDFSFile(sliderFileSystem, pathStr); - log.info("Slider agent package is properly installed at " + pathStr); - } catch (FileNotFoundException e) { - log.error("can not find agent package: {}", pathStr, e); - return; - } catch (IOException e) { - log.error("can not open agent package: {}", pathStr, e); - return; - } - } - } - - String pkgTarballPath = getApplicationDefinitionPath(instanceDefinition - .getAppConfOperations()); - try { - validateHDFSFile(sliderFileSystem, pkgTarballPath); - log.info("Application package is properly installed"); - } catch (FileNotFoundException e) { - log.error("can not find application package: {}", pkgTarballPath, e); - } catch (IOException e) { - log.error("can not open application package: {} ", pkgTarballPath, e); - } - } - } - private void actionDiagnosticAll(ActionDiagnosticArgs diagnosticArgs) throws IOException, YarnException { // assign application name from param to each sub diagnostic function @@ -3950,7 +2647,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe if(isUnset(clusterName)){ throw new BadCommandArgumentsException("application name must be provided with --name option"); } - AggregateConf instanceDefinition = fetchInstanceDefinition(clusterName); + AggregateConf instanceDefinition = new AggregateConf(); String imagePath = instanceDefinition.getInternalOperations().get( INTERNAL_APPLICATION_IMAGE_PATH); // if null, it will be uploaded by Slider and thus at slider's path @@ -3966,21 +2663,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe log.info("The path of slider agent tarball on HDFS is: " + imagePath); } - private AggregateConf fetchInstanceDefinition(String clusterName) - throws YarnException, IOException { - SliderClusterOperations clusterOperations; - AggregateConf instanceDefinition = null; - try { - clusterOperations = createClusterOperations(clusterName); - instanceDefinition = clusterOperations.getInstanceDefinition(); - } catch (YarnException | IOException e) { - log.error("Failed to retrieve instance definition from YARN: " - + e.toString()); - throw e; - } - return instanceDefinition; - } - private void actionDiagnosticApplication(ActionDiagnosticArgs diagnosticArgs) throws YarnException, IOException { // not using member variable clustername because we want to place @@ -3988,7 +2670,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // cluster name has to be put behind action String clusterName = diagnosticArgs.name; requireArgumentSet(Arguments.ARG_NAME, clusterName); - AggregateConf instanceDefinition = fetchInstanceDefinition(clusterName); + AggregateConf instanceDefinition = new AggregateConf(); String clusterDir = instanceDefinition.getAppConfOperations() .getGlobalOptions().get(AgentKeys.APP_ROOT); String pkgTarball = getApplicationDefinitionPath(instanceDefinition.getAppConfOperations()); @@ -4442,7 +3124,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe @VisibleForTesting public int actionLookup(ActionLookupArgs args) throws IOException, YarnException { - verifyBindingsDefined(); try { ApplicationId id = ConverterUtils.toApplicationId(args.id); ApplicationReport report = yarnClient.getApplicationReport(id); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java index c6cc2d0d54a..2bb224b3c72 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java @@ -23,6 +23,7 @@ import org.apache.hadoop.service.Service; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.slider.api.resource.Application; import org.apache.slider.api.types.NodeInformationList; import org.apache.slider.api.types.SliderInstanceDescription; import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; @@ -61,15 +62,8 @@ import java.util.Map; * Stability: evolving */ public interface SliderClientAPI extends Service { - /** - * Destroy a cluster. There's two race conditions here - * #1 the cluster is started between verifying that there are no live - * clusters of that name. - */ - int actionDestroy(String clustername, ActionDestroyArgs destroyArgs) - throws YarnException, IOException; - int actionDestroy(String clustername) throws YarnException, + void actionDestroy(String clustername) throws YarnException, IOException; /** @@ -87,18 +81,6 @@ public interface SliderClientAPI extends Service { AbstractClientProvider createClientProvider(String provider) throws SliderException; - /** - * Build up the cluster specification/directory - * - * @param clustername cluster name - * @param buildInfo the arguments needed to build the cluster - * @throws YarnException Yarn problems - * @throws IOException other problems - * @throws BadCommandArgumentsException bad arguments. - */ - int actionBuild(String clustername, - AbstractClusterBuildingActionArgs buildInfo) throws YarnException, IOException; - /** * Upload keytab to a designated sub-directory of the user home directory * @@ -188,24 +170,7 @@ public interface SliderClientAPI extends Service { */ int actionUpgrade(String clustername, ActionUpgradeArgs buildInfo) - throws YarnException, IOException; - - /** - * Get the report of a this application - * @return the app report or null if it could not be found. - * @throws IOException - * @throws YarnException - */ - ApplicationReport getApplicationReport() - throws IOException, YarnException; - - /** - * Kill the submitted application via YARN - * @throws YarnException - * @throws IOException - */ - boolean forceKillApplication(String reason) - throws YarnException, IOException; + throws YarnException, IOException; /** * Implement the list action: list all nodes @@ -213,30 +178,8 @@ public interface SliderClientAPI extends Service { */ int actionList(String clustername, ActionListArgs args) throws IOException, YarnException; - /** - * Enumerate slider instances for the current user, and the - * most recent app report, where available. - * @param listOnlyInState boolean to indicate that the instances should - * only include those in a YARN state - * minAppState <= currentState <= maxAppState - * - * @param minAppState minimum application state to include in enumeration. - * @param maxAppState maximum application state to include - * @return a map of application instance name to description - * @throws IOException Any IO problem - * @throws YarnException YARN problems - */ - Map enumSliderInstances( - boolean listOnlyInState, - YarnApplicationState minAppState, - YarnApplicationState maxAppState) - throws IOException, YarnException; - /** - * Implement the islive action: probe for a cluster of the given name existing - * @return exit code - */ - int actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException; + void actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException; /** * Test for a cluster existing probe for a cluster of the given name existing @@ -288,7 +231,7 @@ public interface SliderClientAPI extends Service { * @throws YarnException * @throws IOException */ - String actionStatus(String clustername) throws YarnException, IOException; + Application actionStatus(String clustername) throws YarnException, IOException; /** * Version Details @@ -303,13 +246,13 @@ public interface SliderClientAPI extends Service { * @param freezeArgs arguments to the stop * @return EXIT_SUCCESS if the cluster was not running by the end of the operation */ - int actionFreeze(String clustername, ActionFreezeArgs freezeArgs) + void actionStop(String clustername, ActionFreezeArgs freezeArgs) throws YarnException, IOException; /** * Restore a cluster */ - int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException; + int actionStart(String clustername, ActionThawArgs thaw) throws YarnException, IOException; /** * Registry operation diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java index 258ef31b004..48393955725 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java @@ -60,6 +60,10 @@ import java.util.List; import java.util.Map; import java.util.Set; +//TODO, Remove this class and YarnAppListClient +// why do we need so many yarn client wrappers ? +// - yarn client already provides most of functionality already + /** * A class that extends visibility to some of the YarnClientImpl * members and data structures, and factors out pure-YARN operations @@ -68,25 +72,6 @@ import java.util.Set; public class SliderYarnClientImpl extends YarnClientImpl { protected static final Logger log = LoggerFactory.getLogger(SliderYarnClientImpl.class); - /** - * Keyword to use in the {@link #emergencyForceKill(String)} - * operation to force kill all application instances belonging - * to a specific user - */ - public static final String KILL_ALL = "all"; - - @Override - protected void serviceInit(Configuration conf) throws Exception { - InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(conf); - if (!SliderUtils.isAddressDefined(clientRpcAddress)) { - // address isn't known; fail fast - throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS - + " value:" + conf.get(YarnConfiguration.RM_ADDRESS) - + " - see https://wiki.apache.org/hadoop/UnsetHostnameOrPort"); - } - super.serviceInit(conf); - } - /** * Get the RM Client RPC interface * @return an RPC interface valid after initialization and authentication @@ -95,52 +80,6 @@ public class SliderYarnClientImpl extends YarnClientImpl { return rmClient; } - /** - * List Slider runninginstances belonging to a specific user. - * @deprecated use {@link #listDeployedInstances(String)} - * @param user user: "" means all users - * @return a possibly empty list of Slider AMs - */ - public List listInstances(String user) - throws YarnException, IOException { - return listDeployedInstances(user); - } - - /** - * List Slider deployedinstances belonging to a specific user. - *

- * Deployed means: known about in the YARN cluster; it will include - * any that are in the failed/finished state, as well as those queued - * for starting. - * @param user user: "" means all users - * @return a possibly empty list of Slider AMs - */ - public List listDeployedInstances(String user) - throws YarnException, IOException { - return listDeployedInstances(user, null); - } - - /** - * List Slider deployedinstances belonging to a specific user in a - * given set of states. - *

- * Deployed means: known about in the YARN cluster; it will include all apps - * in the specified set of states. - * - * @param user - * user: "" means all users - * @param appStates - * filter by a set of YarnApplicationState - * @return a possibly empty list of Slider AMs - * @throws YarnException - * @throws IOException - */ - public List listDeployedInstances(String user, - EnumSet appStates) - throws YarnException, IOException { - return listDeployedInstances(user, appStates, null); - } - /** * List Slider deployedinstances belonging to a specific user in a * given set of states and filtered by an application name tag. @@ -178,21 +117,6 @@ public class SliderYarnClientImpl extends YarnClientImpl { } return results; } - - /** - * find all instances of a specific app -if there is more than one in the - * YARN cluster, - * this returns them all - * @param user user; use "" for all users - * @param appname application name - * @return the list of all matching application instances - */ - public List findAllInstances(String user, String appname) - throws IOException, YarnException { - Preconditions.checkArgument(appname != null, "Null application name"); - - return listDeployedInstances(user, null, appname); - } /** * Helper method to determine if a cluster application is running -or @@ -206,122 +130,6 @@ public class SliderYarnClientImpl extends YarnClientImpl { return app.getYarnApplicationState().ordinal() <= YarnApplicationState.RUNNING.ordinal(); } - - /** - * Kill a running application - * @param applicationId app Id - * @param reason reason: reason for log - * @return the response - * @throws YarnException YARN problems - * @throws IOException IO problems - */ - public KillApplicationResponse killRunningApplication(ApplicationId applicationId, - String reason) - throws YarnException, IOException { - Preconditions.checkArgument(applicationId != null, "Null application Id"); - log.info("Killing application {} - {}", applicationId.getClusterTimestamp(), - reason); - KillApplicationRequest request = - Records.newRecord(KillApplicationRequest.class); - request.setApplicationId(applicationId); - return getRmClient().forceKillApplication(request); - } - - private String getUsername() throws IOException { - return UserGroupInformation.getCurrentUser().getShortUserName(); - } - - /** - * Force kill a yarn application by ID. No niceties here - * @param applicationId app Id. "all" means "kill all instances of the current user - * - */ - public void emergencyForceKill(String applicationId) - throws YarnException, IOException { - - Preconditions.checkArgument(StringUtils.isNotEmpty(applicationId), - "Null/empty application Id"); - - if (KILL_ALL.equals(applicationId)) { - // user wants all instances killed - String user = getUsername(); - log.info("Killing all applications belonging to {}", user); - Collection instances = listDeployedInstances(user, - SliderUtils.getAllLiveAppStates()); - for (ApplicationReport instance : instances) { - ApplicationId appId = instance.getApplicationId(); - log.info("Killing Application {}", appId); - killRunningApplication(appId, "forced kill"); - } - } else { - ApplicationId appId = ConverterUtils.toApplicationId(applicationId); - - log.info("Killing Application {}", applicationId); - - killRunningApplication(appId, "forced kill"); - } - } - - /** - * Monitor the submitted application for reaching the requested state. - * Will also report if the app reaches a later state (failed, killed, etc) - * Kill application if duration!= null & time expires. - * @param appId Application Id of application to be monitored - * @param duration how long to wait -must be more than 0 - * @param desiredState desired state. - * @return the application report -null on a timeout - * @throws YarnException - * @throws IOException - */ - public ApplicationReport monitorAppToState( - ApplicationId appId, YarnApplicationState desiredState, Duration duration) - throws YarnException, IOException { - - if (appId == null) { - throw new BadCommandArgumentsException("null application ID"); - } - if (duration.limit <= 0) { - throw new BadCommandArgumentsException("Invalid monitoring duration"); - } - log.debug("Waiting {} millis for app to reach state {} ", - duration.limit, - desiredState); - duration.start(); - try { - while (true) { - // Get application report for the appId we are interested in - - ApplicationReport r = getApplicationReport(appId); - - log.debug("queried status is\n{}", - new SliderUtils.OnDemandReportStringifier(r)); - - YarnApplicationState state = r.getYarnApplicationState(); - if (state.ordinal() >= desiredState.ordinal()) { - log.debug("App in desired state (or higher) :{}", state); - return r; - } - if (duration.getLimitExceeded()) { - log.debug( - "Wait limit of {} millis to get to state {}, exceeded; app status\n {}", - duration.limit, - desiredState, - new SliderUtils.OnDemandReportStringifier(r)); - return null; - } - - // sleep 1s. - try { - Thread.sleep(1000); - } catch (InterruptedException ignored) { - log.debug("Thread sleep in monitoring loop interrupted"); - } - } - } finally { - duration.close(); - } - } - /** * find all live instances of a specific app -if there is >1 in the cluster, * this returns them all. State should be running or less diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java index a0073264c2f..3b5147ff368 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java @@ -112,17 +112,6 @@ public class SliderApplicationIpcClient implements SliderApplicationApi { } } - - @Override - public void putDesiredResources(ConfTree updated) throws IOException { - try { - operations.flex(updated); - } catch (IOException e) { - throw convert(e); - } - } - - @Override public AggregateConf getResolvedModel() throws IOException { try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java index 702233a0678..623b8b08a35 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java @@ -25,6 +25,8 @@ import org.apache.slider.api.ClusterNode; import org.apache.slider.api.SliderClusterProtocol; import org.apache.slider.api.StateValues; import org.apache.slider.api.proto.Messages; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.ContainerInformation; @@ -39,6 +41,7 @@ import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.exceptions.WaitTimeoutException; import org.apache.slider.core.persist.ConfTreeSerDeser; +import org.apache.slider.core.persist.JsonSerDeser; import org.codehaus.jackson.JsonParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,6 +64,8 @@ public class SliderClusterOperations { log = LoggerFactory.getLogger(SliderClusterOperations.class); private final SliderClusterProtocol appMaster; + private static final JsonSerDeser jsonSerDeser = + new JsonSerDeser(Application.class); private static final Messages.EmptyPayloadProto EMPTY; static { EMPTY = Messages.EmptyPayloadProto.newBuilder().build(); @@ -130,48 +135,20 @@ public class SliderClusterOperations { * Connect to a live cluster and get its current state * @return its description */ - public ClusterDescription getClusterDescription() - throws YarnException, IOException { - + public Application getApplication() throws YarnException, IOException { Messages.GetJSONClusterStatusRequestProto req = Messages.GetJSONClusterStatusRequestProto.newBuilder().build(); Messages.GetJSONClusterStatusResponseProto resp = appMaster.getJSONClusterStatus(req); String statusJson = resp.getClusterSpec(); try { - return ClusterDescription.fromJson(statusJson); + return jsonSerDeser.fromJson(statusJson); } catch (JsonParseException e) { - log.error("Exception " + e + " parsing:\n" + statusJson, e); + log.error("Error when parsing app json file", e); throw e; } } - /** - * Get the AM instance definition. - *

- * See {@link SliderClusterProtocol#getInstanceDefinition(Messages.GetInstanceDefinitionRequestProto)} - * @return current slider AM aggregate definition - * @throws YarnException - * @throws IOException - */ - public AggregateConf getInstanceDefinition() - throws YarnException, IOException { - Messages.GetInstanceDefinitionRequestProto.Builder builder = - Messages.GetInstanceDefinitionRequestProto.newBuilder(); - - Messages.GetInstanceDefinitionRequestProto request = builder.build(); - Messages.GetInstanceDefinitionResponseProto response = - appMaster.getInstanceDefinition(request); - - ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser(); - - ConfTree internal = confTreeSerDeser.fromJson(response.getInternal()); - ConfTree resources = confTreeSerDeser.fromJson(response.getResources()); - ConfTree app = confTreeSerDeser.fromJson(response.getApplication()); - AggregateConf instanceDefinition = - new AggregateConf(resources, app, internal); - return instanceDefinition; - } /** * Kill a container * @param id container ID @@ -315,22 +292,14 @@ public class SliderClusterOperations { return state; } - /** - * Flex operation - * @param resources new resources - * @return the response - * @throws IOException - */ - public boolean flex(ConfTree resources) throws IOException { - Messages.FlexClusterRequestProto request = - Messages.FlexClusterRequestProto.newBuilder() - .setClusterSpec(resources.toJson()) - .build(); - Messages.FlexClusterResponseProto response = appMaster.flexCluster(request); - return response.getResponse(); + public void flex(Component component) throws IOException{ + Messages.FlexComponentRequestProto request = + Messages.FlexComponentRequestProto.newBuilder() + .setNumberOfContainers(component.getNumberOfContainers().intValue()) + .setName(component.getName()).build(); + appMaster.flexComponent(request); } - /** * Commit (possibly delayed) AM suicide * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java index 573ef64ae40..4c376e0da69 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java @@ -176,29 +176,6 @@ public class SliderApplicationApiRestClient extends BaseRestClient return new ConfTreeOperations(resource); } - @Override - public void putDesiredResources(ConfTree updated) throws IOException { - WebResource resource = applicationResource(MODEL_DESIRED_RESOURCES); - try { - - // put operation. The result is discarded; it does help validate - // that the operation returned a JSON data structure as well as a 200 - // response. - - resource.accept(MediaType.APPLICATION_JSON_TYPE) - .type(MediaType.APPLICATION_JSON_TYPE) - .entity(updated) - .put(ConfTree.class); - } catch (ClientHandlerException ex) { - throw ExceptionConverter.convertJerseyException("PUT", - resource.getURI().toString(), - ex); - } catch (UniformInterfaceException ex) { - throw ExceptionConverter.convertJerseyException("PUT", - resource.getURI().toString(), ex); - } - } - @Override public AggregateConf getResolvedModel() throws IOException { return getApplicationResource(MODEL_RESOLVED, AggregateConf.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 4bf1b5bb954..9a4fa6c4f1c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -18,6 +18,8 @@ package org.apache.slider.common; +import org.apache.hadoop.yarn.api.ApplicationConstants; + import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -50,7 +52,7 @@ public interface SliderKeys extends SliderXmlConfKeys { String SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK = "slider_dep"; String SLIDER_DEPENDENCY_HDP_PARENT_DIR = "/hdp"; String SLIDER_DEPENDENCY_DIR = "/apps/%s/slider"; - String SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME = "slider"; + String SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME = "slider-dep"; String SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT = ".tar.gz"; String SLIDER_DEPENDENCY_DIR_PERMISSIONS = "755"; @@ -181,7 +183,7 @@ public interface SliderKeys extends SliderXmlConfKeys { /** * name of generated dir for this conf: {@value} */ - String SUBMITTED_CONF_DIR = "confdir"; + String SUBMITTED_CONF_DIR = "conf"; /** * Slider AM log4j file name : {@value} @@ -227,7 +229,7 @@ public interface SliderKeys extends SliderXmlConfKeys { */ String ADDONS_DIR = "addons"; - String SLIDER_JAR = "slider.jar"; + String SLIDER_JAR = "slider-core.jar"; String JCOMMANDER_JAR = "jcommander.jar"; String GSON_JAR = "gson.jar"; String DEFAULT_APP_PKG = "appPkg.zip"; @@ -238,7 +240,7 @@ public interface SliderKeys extends SliderXmlConfKeys { String STDERR_AM = "slider-err.txt"; String DEFAULT_GC_OPTS = ""; - String HADOOP_USER_NAME = "HADOOP_USER_NAME"; + String HADOOP_USER_NAME = ApplicationConstants.Environment.USER.toString(); String HADOOP_PROXY_USER = "HADOOP_PROXY_USER"; String SLIDER_PASSPHRASE = "SLIDER_PASSPHRASE"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java index 72dd44f7752..b6668346582 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java @@ -73,6 +73,9 @@ public interface SliderXmlConfKeys { int DEFAULT_YARN_QUEUE_PRIORITY = 1; + String KEY_AM_RESOURCE_MEM = "slider.am.resource.memory"; + long DEFAULT_KEY_AM_RESOURCE_MEM = 1024; + /** * The slider base path: {@value} * Defaults to HomeDir/.slider diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java index 63ccff845c4..6dd61fabbce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java @@ -69,7 +69,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments { /** -D name=value - Define an HBase configuration option which overrides any options in + Define an configuration option which overrides any options in the configuration XML files of the image or in the image configuration directory. The values will be persisted. Configuration options are only passed to the cluster when creating or reconfiguring a cluster. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java index 1d28c7841dc..2d471ed88f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java @@ -35,186 +35,13 @@ import java.util.Map; * Abstract Action to build things; shares args across build and * list */ -public abstract class AbstractClusterBuildingActionArgs extends - AbstractActionArgs { - - /** - * Declare the image configuration directory to use when creating or - * reconfiguring a slider cluster. The path must be on a filesystem visible - * to all nodes in the YARN cluster. Only one configuration directory can - * be specified. - */ - @Parameter(names = ARG_CONFDIR, - description = "Path to cluster configuration directory in HDFS", - converter = PathArgumentConverter.class) - public Path confdir; - - @Parameter(names = ARG_ZKPATH, - description = "Zookeeper path for the application") - public String appZKPath; - - @Parameter(names = ARG_ZKHOSTS, - description = "comma separated list of the Zookeeper hosts") - public String zkhosts; - - /** - * --image path - * the full path to a .tar or .tar.gz path containing an HBase image. - */ - @Parameter(names = ARG_IMAGE, - description = "The full path to a .tar or .tar.gz path containing the application", - converter = PathArgumentConverter.class) - public Path image; - - @Parameter(names = ARG_APP_HOME, - description = "Home directory of a pre-installed application") - public String appHomeDir; - - @Parameter(names = ARG_PROVIDER, - description = "Provider of the specific cluster application") - public String provider = SliderProviderFactory.DEFAULT_CLUSTER_TYPE; - - @Parameter(names = {ARG_PACKAGE}, - description = "URI to a slider package") - public String packageURI; - - @Parameter(names = {ARG_RESOURCES}, - description = "File defining the resources of this instance") - public File resources; - - @Parameter(names = {ARG_TEMPLATE}, - description = "Template application configuration") - public File template; - - @Parameter(names = {ARG_METAINFO}, - description = "Application meta info file") - public File appMetaInfo; - - @Parameter(names = {ARG_METAINFO_JSON}, - description = "Application meta info JSON blob") - public String appMetaInfoJson; - - @Parameter(names = {ARG_APPDEF}, - description = "Application def (folder or a zip package)") - public File appDef; - - @Parameter(names = {ARG_QUEUE}, - description = "Queue to submit the application") +public abstract class AbstractClusterBuildingActionArgs + extends AbstractActionArgs { + @Parameter(names = { + ARG_QUEUE }, description = "Queue to submit the application") public String queue; - @Parameter(names = {ARG_LIFETIME}, - description = "Lifetime of the application from the time of request") + @Parameter(names = { + ARG_LIFETIME }, description = "Lifetime of the application from the time of request") public long lifetime; - - @ParametersDelegate - public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate(); - - @ParametersDelegate - public AddonArgsDelegate addonDelegate = new AddonArgsDelegate(); - - - @ParametersDelegate - public AppAndResouceOptionArgsDelegate optionsDelegate = - new AppAndResouceOptionArgsDelegate(); - - - public Map getOptionsMap() throws - BadCommandArgumentsException { - return optionsDelegate.getOptionsMap(); - } - - /** - * Get the role heap mapping (may be empty, but never null) - * @return role heap mapping - * @throws BadCommandArgumentsException parse problem - */ - public Map> getCompOptionMap() throws - BadCommandArgumentsException { - return optionsDelegate.getCompOptionMap(); - } - - - public Map getResourceOptionsMap() throws - BadCommandArgumentsException { - return optionsDelegate.getResourceOptionsMap(); - } - - /** - * Get the role heap mapping (may be empty, but never null) - * @return role heap mapping - * @throws BadCommandArgumentsException parse problem - */ - public Map> getResourceCompOptionMap() throws - BadCommandArgumentsException { - return optionsDelegate.getResourceCompOptionMap(); - } - - @VisibleForTesting - public List getComponentTuples() { - return componentDelegate.getComponentTuples(); - } - - /** - * Get the role mapping (may be empty, but never null) - * @return role mapping - * @throws BadCommandArgumentsException parse problem - */ - public Map getComponentMap() throws - BadCommandArgumentsException { - return componentDelegate.getComponentMap(); - } - - @VisibleForTesting - public List getAddonTuples() { - return addonDelegate.getAddonTuples(); - } - - /** - * Get the list of addons (may be empty, but never null) - */ - public Map getAddonMap() throws - BadCommandArgumentsException { - return addonDelegate.getAddonMap(); - } - - public Path getConfdir() { - return confdir; - } - - public String getAppZKPath() { - return appZKPath; - } - - public String getZKhosts() { - return zkhosts; - } - - public Path getImage() { - return image; - } - - public String getAppHomeDir() { - return appHomeDir; - } - - public String getProvider() { - return provider; - } - - public ConfTree buildAppOptionsConfTree() throws - BadCommandArgumentsException { - return buildConfTree(getOptionsMap()); - } - - public ConfTree buildResourceOptionsConfTree() throws - BadCommandArgumentsException { - return buildConfTree(getResourceOptionsMap()); - } - - protected ConfTree buildConfTree(Map optionsMap) throws - BadCommandArgumentsException { - ConfTree confTree = new ConfTree(); - confTree.global.putAll(optionsMap); - return confTree; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java deleted file mode 100644 index 1a182d1414d..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionBuildArgs.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.common.params; - -import com.beust.jcommander.Parameters; - -@Parameters(commandNames = {SliderActions.ACTION_BUILD}, - commandDescription = SliderActions.DESCRIBE_ACTION_BUILD) - -public class ActionBuildArgs extends AbstractClusterBuildingActionArgs { - - @Override - public String getActionName() { - return SliderActions.ACTION_BUILD; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java index e70f30ad620..c8cac65e1fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java @@ -18,6 +18,7 @@ package org.apache.slider.common.params; +import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.beust.jcommander.ParametersDelegate; @@ -26,34 +27,19 @@ import java.io.File; @Parameters(commandNames = {SliderActions.ACTION_CREATE}, commandDescription = SliderActions.DESCRIBE_ACTION_CREATE) -public class ActionCreateArgs extends AbstractClusterBuildingActionArgs - implements WaitTimeAccessor, LaunchArgsAccessor { - +public class ActionCreateArgs extends AbstractClusterBuildingActionArgs { + + @Parameter(names = {ARG_APPDEF}, + description = "Template application definition file in JSON format.") + public File appDef; + + public File getAppDef() { + return appDef; + } + @Override public String getActionName() { return SliderActions.ACTION_CREATE; } - - @ParametersDelegate - LaunchArgsDelegate launchArgs = new LaunchArgsDelegate(); - - @Override - public File getOutputFile() { - return launchArgs.getOutputFile(); - } - - @Override - public String getRmAddress() { - return launchArgs.getRmAddress(); - } - - @Override - public int getWaittime() { - return launchArgs.getWaittime(); - } - - @Override - public void setWaittime(int waittime) { - launchArgs.setWaittime(waittime); - } } + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java index 725973e0468..c565484cf2c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFlexArgs.java @@ -18,37 +18,31 @@ package org.apache.slider.common.params; +import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import com.beust.jcommander.ParametersDelegate; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; - -import java.util.List; -import java.util.Map; @Parameters(commandNames = {SliderActions.ACTION_FLEX}, commandDescription = SliderActions.DESCRIBE_ACTION_FLEX) public class ActionFlexArgs extends AbstractActionArgs { + @Parameter(names = {ARG_COMPONENT}, + description = "component name") + String componentName; + + @Parameter(names = {ARG_COUNT}, + description = "number of containers>") + long numberOfContainers; + @Override public String getActionName() { return SliderActions.ACTION_FLEX; } - - @ParametersDelegate - public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate(); - /** - * Get the component mapping (may be empty, but never null) - * @return mapping - * @throws BadCommandArgumentsException parse problem - */ - public Map getComponentMap() throws BadCommandArgumentsException { - return componentDelegate.getComponentMap(); + public String getComponent() { + return componentName; } - - public List getComponentTuples() { - return componentDelegate.getComponentTuples(); + public long getNumberOfContainers() { + return numberOfContainers; } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java index e3085d9ba24..f3cc6ef1e26 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionFreezeArgs.java @@ -22,14 +22,14 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import com.beust.jcommander.ParametersDelegate; -@Parameters(commandNames = {SliderActions.ACTION_FREEZE}, +@Parameters(commandNames = {SliderActions.ACTION_STOP }, commandDescription = SliderActions.DESCRIBE_ACTION_FREEZE) public class ActionFreezeArgs extends AbstractActionArgs implements WaitTimeAccessor { @Override public String getActionName() { - return SliderActions.ACTION_FREEZE; + return SliderActions.ACTION_STOP; } public static final String FREEZE_COMMAND_ISSUED = "stop command issued"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java index 2bd856f7eba..04988c9eebb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java @@ -24,7 +24,7 @@ import com.beust.jcommander.ParametersDelegate; import java.io.File; -@Parameters(commandNames = {SliderActions.ACTION_THAW}, +@Parameters(commandNames = {SliderActions.ACTION_START }, commandDescription = SliderActions.DESCRIBE_ACTION_THAW) public class ActionThawArgs extends AbstractActionArgs implements WaitTimeAccessor, @@ -33,7 +33,7 @@ public class ActionThawArgs extends AbstractActionArgs implements @Override public String getActionName() { - return SliderActions.ACTION_THAW; + return SliderActions.ACTION_START; } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java index 6ef51b2040e..18aa1f5474c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java @@ -18,56 +18,31 @@ package org.apache.slider.common.params; -import java.io.File; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + import java.util.ArrayList; import java.util.List; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.beust.jcommander.ParametersDelegate; - @Parameters(commandNames = { SliderActions.ACTION_UPGRADE }, commandDescription = SliderActions.DESCRIBE_ACTION_UPGRADE) -public class ActionUpgradeArgs extends AbstractClusterBuildingActionArgs - implements WaitTimeAccessor, LaunchArgsAccessor { +public class ActionUpgradeArgs extends AbstractActionArgs { @Override public String getActionName() { return SliderActions.ACTION_UPGRADE; } - - @ParametersDelegate - LaunchArgsDelegate launchArgs = new LaunchArgsDelegate(); - - @Override - public File getOutputFile() { - return launchArgs.getOutputFile(); - } - - @Override - public String getRmAddress() { - return launchArgs.getRmAddress(); - } - - @Override - public int getWaittime() { - return launchArgs.getWaittime(); - } - - @Override - public void setWaittime(int waittime) { - launchArgs.setWaittime(waittime); - } - - @Parameter(names={ARG_CONTAINERS}, variableArity = true, - description = "stop specific containers") - public List containers = new ArrayList<>(0); - - @Parameter(names={ARG_COMPONENTS}, variableArity = true, - description = "stop all containers of specific components") - public List components = new ArrayList<>(0); - - @Parameter(names = {ARG_FORCE}, - description = "force spec upgrade operation") - public boolean force; + +// TODO upgrade container +// @Parameter(names={ARG_CONTAINERS}, variableArity = true, +// description = "stop specific containers") +// public List containers = new ArrayList<>(0); +// +// @Parameter(names={ARG_COMPONENTS}, variableArity = true, +// description = "stop all containers of specific components") +// public List components = new ArrayList<>(0); +// +// @Parameter(names = {ARG_FORCE}, +// description = "force spec upgrade operation") +// public boolean force; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java index cbf7e59dbb1..45c1fbda9c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java @@ -36,6 +36,7 @@ public interface Arguments { String ARG_CLIENT = "--client"; String ARG_CONFDIR = "--appconf"; String ARG_COMPONENT = "--component"; + String ARG_COUNT = "--count"; String ARG_COMPONENT_SHORT = "--comp"; String ARG_COMPONENTS = "--components"; String ARG_COMP_OPT= "--compopt"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java index 4016cc99c49..abd2ce70321 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java @@ -53,7 +53,6 @@ public class ClientArgs extends CommonArgs { // ========================================================= private final ActionAMSuicideArgs actionAMSuicideArgs = new ActionAMSuicideArgs(); - private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs(); private final ActionClientArgs actionClientArgs = new ActionClientArgs(); private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs(); private final ActionDependencyArgs actionDependencyArgs = new ActionDependencyArgs(); @@ -96,7 +95,6 @@ public class ClientArgs extends CommonArgs { addActions( actionAMSuicideArgs, - actionBuildArgs, actionClientArgs, actionCreateArgs, actionDependencyArgs, @@ -155,10 +153,6 @@ public class ClientArgs extends CommonArgs { return actionAMSuicideArgs; } - public ActionBuildArgs getActionBuildArgs() { - return actionBuildArgs; - } - public ActionInstallPackageArgs getActionInstallPackageArgs() { return actionInstallPackageArgs; } public ActionClientArgs getActionClientArgs() { return actionClientArgs; } @@ -256,23 +250,17 @@ public class ClientArgs extends CommonArgs { action = ACTION_HELP; } switch (action) { - case ACTION_BUILD: - bindCoreAction(actionBuildArgs); - //its a builder, so set those actions too - buildingActionArgs = actionBuildArgs; - break; - case ACTION_CREATE: bindCoreAction(actionCreateArgs); //its a builder, so set those actions too buildingActionArgs = actionCreateArgs; break; - case ACTION_FREEZE: + case ACTION_STOP: bindCoreAction(actionFreezeArgs); break; - case ACTION_THAW: + case ACTION_START: bindCoreAction(actionThawArgs); break; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java index f9516d18d70..de65954bc60 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java @@ -43,7 +43,7 @@ public class SliderAMArgs extends CommonArgs { * This is the URI in the FS to the Slider cluster; the conf file (and any * other cluster-specifics) can be picked up here */ - public String getSliderClusterURI() { + public String getAppDefDir() { return createAction.sliderClusterURI; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java index 204ad9a34bc..82e5903a969 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java @@ -36,7 +36,7 @@ public interface SliderActions { String ACTION_ECHO = "echo"; String ACTION_EXISTS = "exists"; String ACTION_FLEX = "flex"; - String ACTION_FREEZE = "stop"; + String ACTION_STOP = "stop"; String ACTION_HELP = "help"; String ACTION_INSTALL_KEYTAB = "install-keytab"; String ACTION_INSTALL_PACKAGE = "install-package"; @@ -53,7 +53,7 @@ public interface SliderActions { String ACTION_RESOLVE = "resolve"; String ACTION_RESOURCE = "resource"; String ACTION_STATUS = "status"; - String ACTION_THAW = "start"; + String ACTION_START = "start"; String ACTION_TOKENS = "tokens"; String ACTION_VERSION = "version"; @@ -68,7 +68,7 @@ public interface SliderActions { String DESCRIBE_ACTION_UPDATE = "Update template for a Slider application"; String DESCRIBE_ACTION_UPGRADE = - "Rolling upgrade/downgrade the application to a newer/previous version"; + "Rolling upgrade/downgrade the component/containerto a newer/previous version"; String DESCRIBE_ACTION_DESTROY = "Destroy a stopped Slider application"; String DESCRIBE_ACTION_EXISTS = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java index c3d6d985024..591931237c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java @@ -511,6 +511,7 @@ public class CoreFileSystem { * @return the parent dir path of slider.tar.gz in HDFS */ public Path getDependencyPath() { + // FIXME: 3/20/17 HDP ??????????? String parentDir = (SliderUtils.isHdp()) ? SliderKeys.SLIDER_DEPENDENCY_HDP_PARENT_DIR + SliderKeys.SLIDER_DEPENDENCY_DIR : SliderKeys.SLIDER_DEPENDENCY_DIR; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java index 294f37ec368..40b07bda995 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java @@ -20,6 +20,7 @@ package org.apache.slider.common.tools; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import java.io.IOException; @@ -29,6 +30,8 @@ import java.io.IOException; */ public class SliderFileSystem extends CoreFileSystem { + Path appDir = null; + public SliderFileSystem(FileSystem fileSystem, Configuration configuration) { super(fileSystem, configuration); @@ -38,5 +41,11 @@ public class SliderFileSystem extends CoreFileSystem { super(configuration); } + public void setAppDir(Path appDir) { + this.appDir = appDir; + } + public Path getAppDir() { + return this.appDir; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 20c78313fe7..c0ef2d45a63 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -499,9 +499,6 @@ public final class SliderUtils { "Source file not a file " + srcFile); } FileSystem destFS = FileSystem.get(destFile.toUri(), conf); - if (destFS.exists(destFile)) { - throw new IOException("Dest file already exists " + destFile); - } FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf); } @@ -1221,6 +1218,29 @@ public final class SliderUtils { return buildEnvMap(roleOpts, null); } + + // Build env map: key -> value; + // value will be replaced by the corresponding value in tokenMap, if any. + public static Map buildEnvMap( + org.apache.slider.api.resource.Configuration conf, + Map tokenMap) { + if (tokenMap == null) { + return conf.getEnv(); + } + Map env = new HashMap<>(); + for (Map.Entry entry : conf.getEnv().entrySet()) { + String key = entry.getKey(); + String val = entry.getValue(); + for (Map.Entry token : tokenMap.entrySet()) { + val = val.replaceAll(Pattern.quote(token.getKey()), + token.getValue()); + } + env.put(key,val); + } + return env; + } + + public static Map buildEnvMap(Map roleOpts, Map tokenMap) { Map env = new HashMap<>(); @@ -1273,8 +1293,8 @@ public final class SliderUtils { * @param clustername cluster name * @throws BadCommandArgumentsException if it is invalid */ - public static void validateClusterName(String clustername) throws - BadCommandArgumentsException { + public static void validateClusterName(String clustername) + throws BadCommandArgumentsException { if (!isClusternameValid(clustername)) { throw new BadCommandArgumentsException( "Illegal cluster name: " + clustername); @@ -1603,14 +1623,12 @@ public final class SliderUtils { * @param sliderConfDir relative path to the dir containing slider config * options to put on the classpath -or null * @param libdir directory containing the JAR files - * @param config the configuration * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use * (and hence the current classpath should be used, not anything built up) * @return a classpath */ public static ClasspathConstructor buildClasspath(String sliderConfDir, String libdir, - Configuration config, SliderFileSystem sliderFileSystem, boolean usingMiniMRCluster) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java deleted file mode 100644 index 7190c3acdb5..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.client.api.YarnClientApplication; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.util.Records; -import org.apache.slider.client.SliderYarnClientImpl; -import org.apache.slider.common.tools.CoreFileSystem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Map; -import java.util.Set; - -public class AppMasterLauncher extends AbstractLauncher { - - - private static final Logger log = - LoggerFactory.getLogger(AppMasterLauncher.class); - - public final YarnClientApplication application; - public final String name; - public final String type; - public final ApplicationSubmissionContext submissionContext; - public final ApplicationId appId; - public final boolean secureCluster; - private int maxAppAttempts = 0; - private boolean keepContainersOverRestarts = true; - private String queue = YarnConfiguration.DEFAULT_QUEUE_NAME; - private int priority = 1; - private final Resource resource = Records.newRecord(Resource.class); - private final SliderYarnClientImpl yarnClient; - private Long submitTime; - - /** - * Build the AM Launcher - * @param name app name - * @param type application type - * @param conf hadoop config - * @param fs filesystem binding - * @param yarnClient yarn client - * @param secureCluster flag to indicate secure cluster - * @param options map of options. All values are extracted in this constructor only - * @param resourceGlobalOptions global options - * @param applicationTags any app tags - * @param credentials initial set of credentials - * @throws IOException - * @throws YarnException - */ - public AppMasterLauncher(String name, - String type, - Configuration conf, - CoreFileSystem fs, - SliderYarnClientImpl yarnClient, - boolean secureCluster, - Map options, - Map resourceGlobalOptions, - Set applicationTags, - Credentials credentials) throws IOException, YarnException { - super(conf, fs, credentials); - this.yarnClient = yarnClient; - this.application = yarnClient.createApplication(); - this.name = name; - this.type = type; - this.secureCluster = secureCluster; - - submissionContext = application.getApplicationSubmissionContext(); - appId = submissionContext.getApplicationId(); - // set the application name; - submissionContext.setApplicationName(name); - // app type used in service enum; - submissionContext.setApplicationType(type); - if (!applicationTags.isEmpty()) { - submissionContext.setApplicationTags(applicationTags); - } - submissionContext.setNodeLabelExpression(extractLabelExpression(options)); - - extractAmRetryCount(submissionContext, resourceGlobalOptions); - extractResourceRequirements(resource, options); - extractLogAggregationContext(resourceGlobalOptions); - } - - public void setMaxAppAttempts(int maxAppAttempts) { - this.maxAppAttempts = maxAppAttempts; - } - - public void setKeepContainersOverRestarts(boolean keepContainersOverRestarts) { - this.keepContainersOverRestarts = keepContainersOverRestarts; - } - - - public Resource getResource() { - return resource; - } - - public void setMemory(int memory) { - resource.setMemory(memory); - } - - public void setVirtualCores(int cores) { - resource.setVirtualCores(cores); - } - - public ApplicationId getApplicationId() { - return appId; - } - - public int getMaxAppAttempts() { - return maxAppAttempts; - } - - public boolean isKeepContainersOverRestarts() { - return keepContainersOverRestarts; - } - - public String getQueue() { - return queue; - } - - public int getPriority() { - return priority; - } - - public void setQueue(String queue) { - this.queue = queue; - } - - public void setPriority(int priority) { - this.priority = priority; - } - - /** - * Complete the launch context (copy in env vars, etc). - * @return the container to launch - */ - public ApplicationSubmissionContext completeAppMasterLaunch() - throws IOException { - - //queue priority - Priority pri = Records.newRecord(Priority.class); - pri.setPriority(priority); - submissionContext.setPriority(pri); - - // Set the queue to which this application is to be submitted in the RM - // Queue for App master - - submissionContext.setQueue(queue); - - - //container requirements - submissionContext.setResource(resource); - submissionContext.setLogAggregationContext(logAggregationContext); - - if (keepContainersOverRestarts) { - log.debug("Requesting cluster stays running over AM failure"); - submissionContext.setKeepContainersAcrossApplicationAttempts(true); - } - - if (maxAppAttempts > 0) { - log.debug("Setting max AM attempts to {}", maxAppAttempts); - submissionContext.setMaxAppAttempts(maxAppAttempts); - } - - if (secureCluster) { - //tokens - log.debug("Credentials: {}", - CredentialUtils.dumpTokens(getCredentials(), "\n")); - - } else { - propagateUsernameInInsecureCluster(); - } - completeContainerLaunch(); - submissionContext.setAMContainerSpec(containerLaunchContext); - return submissionContext; - } - - /** - * Submit the application. - * @return a launched application representing the submitted application - * @throws IOException - * @throws YarnException - */ - public LaunchedApplication submitApplication() throws IOException, YarnException { - completeAppMasterLaunch(); - log.info("Submitting application to Resource Manager"); - ApplicationId applicationId = - yarnClient.submitApplication(submissionContext); - // implicit success; record the time - submitTime = System.currentTimeMillis(); - return new LaunchedApplication(applicationId, yarnClient); - } - - /** - * Build a serializable application report. This is a very minimal - * report that contains the application Id, name and type —the information - * available - * @return a data structure which can be persisted - */ - public SerializedApplicationReport createSerializedApplicationReport() { - SerializedApplicationReport sar = new SerializedApplicationReport(); - sar.applicationId = appId.toString(); - sar.name = name; - sar.applicationType = type; - sar.queue = queue; - sar.submitTime = submitTime; - return sar; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java deleted file mode 100644 index 632e3fd867f..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.slider.client.SliderYarnClientImpl; -import org.apache.slider.common.tools.Duration; - -import java.io.IOException; - -/** - * Launched App with logic around it. - */ -public class LaunchedApplication { - - protected final ApplicationId applicationId; - protected final SliderYarnClientImpl yarnClient; - - public LaunchedApplication(ApplicationId applicationId, - SliderYarnClientImpl yarnClient) { - assert applicationId != null; - assert yarnClient != null; - this.applicationId = applicationId; - this.yarnClient = yarnClient; - } - - public LaunchedApplication(SliderYarnClientImpl yarnClient, - ApplicationReport report) { - this.yarnClient = yarnClient; - this.applicationId = report.getApplicationId(); - } - - public ApplicationId getApplicationId() { - return applicationId; - } - - /** - * Monitor the submitted application for reaching the requested state. - * Will also report if the app reaches a later state (failed, killed, etc) - * Kill application if duration!= null & time expires. - * @param duration how long to wait -must be more than 0 - * @param desiredState desired state. - * @return the application report -null on a timeout - * @throws YarnException - * @throws IOException - */ - public ApplicationReport monitorAppToState(YarnApplicationState desiredState, Duration duration) - throws YarnException, IOException { - return yarnClient.monitorAppToState(applicationId, desiredState, duration); - } - - /** - * Kill the submitted application by sending a call to the ASM - * @throws YarnException - * @throws IOException - */ - public boolean forceKill(String reason) - throws YarnException, IOException { - if (applicationId != null) { - yarnClient.killRunningApplication(applicationId, reason); - return true; - } - return false; - } - - /** - * Kill the application - * @return the response - * @throws YarnException YARN problems - * @throws IOException IO problems - */ - public KillApplicationResponse kill(String reason) throws - YarnException, - IOException { - return yarnClient.killRunningApplication(applicationId, reason); - } - - /** - * Get the application report of this application - * @return an application report - * @throws YarnException - * @throws IOException - */ - public ApplicationReport getApplicationReport() - throws YarnException, IOException { - return yarnClient.getApplicationReport(applicationId); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/RunningApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/RunningApplication.java deleted file mode 100644 index 14c522c6a12..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/RunningApplication.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.slider.api.SliderClusterProtocol; -import org.apache.slider.client.SliderYarnClientImpl; -import org.apache.slider.common.SliderExitCodes; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.server.appmaster.rpc.RpcBinder; - -import java.io.IOException; - -import static org.apache.slider.common.Constants.CONNECT_TIMEOUT; -import static org.apache.slider.common.Constants.RPC_TIMEOUT; - -/** - * A running application built from an app report. This one - * can be talked to - */ -public class RunningApplication extends LaunchedApplication { - - private final ApplicationReport applicationReport; - public RunningApplication(SliderYarnClientImpl yarnClient, - ApplicationReport applicationReport) { - super(yarnClient, applicationReport); - this.applicationReport = applicationReport; - } - - public ApplicationReport getApplicationReport() { - return applicationReport; - } - - - /** - * Connect to a Slider AM - * @param app application report providing the details on the application - * @return an instance - * @throws YarnException - * @throws IOException - */ - public SliderClusterProtocol connect(ApplicationReport app) throws - YarnException, - IOException { - - try { - return RpcBinder.getProxy(yarnClient.getConfig(), - yarnClient.getRmClient(), - app, - CONNECT_TIMEOUT, - RPC_TIMEOUT); - } catch (InterruptedException e) { - throw new SliderException(SliderExitCodes.EXIT_TIMED_OUT, - e, - "Interrupted waiting for communications with the Application Master"); - } - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java deleted file mode 100644 index 9eb7d5c516b..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.persist; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.io.Files; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.fs.Path; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; -import org.apache.slider.common.params.Arguments; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.providers.agent.AgentKeys; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Class to prepare and persist app and add-on definitions. - * - * In this case, the app definition and add-on definitions are auto-inferred from the user input rather than explicit - * inclusion of application package in the config. - * - * Processing an app definition involves one or more of the following: - modify appConfig - package definition into a - * temporary folder - upload to HDFS - * - * This class keeps track of all the required operations and allows them to be invoked by build operation - */ -public class AppDefinitionPersister { - private static final Logger log = - LoggerFactory.getLogger(AppDefinitionPersister.class); - - private final SliderFileSystem sliderFileSystem; - private List appDefinitions; - - public AppDefinitionPersister(SliderFileSystem sliderFileSystem) { - this.sliderFileSystem = sliderFileSystem; - appDefinitions = new ArrayList<>(); - } - - - /** - * Process the application package or folder by copying it to the cluster path - * - * @param appDefinition details of application package - * - * @throws BadConfigException - * @throws IOException - */ - private void persistDefinitionPackageOrFolder(AppDefinition appDefinition) - throws BadConfigException, IOException { - if (!appDefinition.appDefPkgOrFolder.canRead()) { - throw new BadConfigException("Pkg/Folder cannot be accessed - " - + appDefinition.appDefPkgOrFolder.getAbsolutePath()); - } - - File src = appDefinition.appDefPkgOrFolder; - String targetName = appDefinition.pkgName; - log.debug("Package name: " + targetName); - if (appDefinition.appDefPkgOrFolder.isDirectory()) { - log.info("Processing app package/folder {} for {}", - appDefinition.appDefPkgOrFolder.getAbsolutePath(), - appDefinition.pkgName); - File tmpDir = Files.createTempDir(); - File zipFile = new File(tmpDir.getCanonicalPath(), File.separator + appDefinition.pkgName); - SliderUtils.zipFolder(appDefinition.appDefPkgOrFolder, zipFile); - src = zipFile; - } - - sliderFileSystem.getFileSystem().copyFromLocalFile( - false, - false, - new Path(src.toURI()), - new Path(appDefinition.targetFolderInFs, targetName)); - } - - public void persistPackages() throws BadConfigException, IOException { - for (AppDefinition appDefinition : appDefinitions) { - persistDefinitionPackageOrFolder(appDefinition); - } - } - - public void processSuppliedDefinitions(String clustername, - AbstractClusterBuildingActionArgs buildInfo, - ConfTreeOperations appConf) - throws BadConfigException, IOException, BadCommandArgumentsException { - // if metainfo is provided add to the app instance - if (buildInfo.appMetaInfo != null || buildInfo.appMetaInfoJson != null) { - if (buildInfo.appMetaInfo != null && buildInfo.appMetaInfoJson != null) { - throw new BadConfigException("Both %s and %s cannot be specified", - Arguments.ARG_METAINFO, Arguments.ARG_METAINFO_JSON); - } - - // Now we know that only one of either file or JSON is used - boolean isFileUsed = buildInfo.appMetaInfo != null ? true : false; - String argUsed = isFileUsed ? Arguments.ARG_METAINFO - : Arguments.ARG_METAINFO_JSON; - - if (buildInfo.appDef != null) { - throw new BadConfigException("Both %s and %s cannot be specified", - argUsed, Arguments.ARG_APPDEF); - } - if (SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) { - throw new BadConfigException( - "%s cannot not be set if %s is specified in the cmd line ", - AgentKeys.APP_DEF, argUsed); - } - - if (isFileUsed) { - if (!buildInfo.appMetaInfo.canRead() || !buildInfo.appMetaInfo.isFile()) { - throw new BadConfigException( - "Path specified with %s either cannot be read or is not a file", - Arguments.ARG_METAINFO); - } - } else { - if (StringUtils.isEmpty(buildInfo.appMetaInfoJson.trim())) { - throw new BadConfigException("Empty string specified with %s", - Arguments.ARG_METAINFO_JSON); - } - } - - File tempDir = Files.createTempDir(); - File pkgSrcDir = new File(tempDir, "default"); - if (!pkgSrcDir.exists() && !pkgSrcDir.mkdirs()) { - throw new IOException("Failed to create directory: " + pkgSrcDir); - } - File destMetaInfo = new File(pkgSrcDir, "metainfo.json"); - if (isFileUsed) { - if (buildInfo.appMetaInfo.getName().endsWith(".xml")) { - Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.xml")); - } else { - Files.copy(buildInfo.appMetaInfo, destMetaInfo); - } - } else { - Files.write( - buildInfo.appMetaInfoJson.getBytes(Charset.forName("UTF-8")), - destMetaInfo); - } - - Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername); - log.info("Using default app def path {}", appDirPath.toString()); - - appDefinitions.add(new AppDefinition(appDirPath, pkgSrcDir, - SliderKeys.DEFAULT_APP_PKG)); - Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG); - appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath); - log.info("Setting app package to {}.", appDefPath); - } - - if (buildInfo.appDef != null) { - if (SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) { - throw new BadConfigException("application.def must not be set if --appdef is provided."); - } - - if (!buildInfo.appDef.exists()) { - throw new BadConfigException("--appdef is not a valid path."); - } - - Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername); - appDefinitions.add(new AppDefinition(appDirPath, buildInfo.appDef, SliderKeys.DEFAULT_APP_PKG)); - Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG); - appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath); - log.info("Setting app package to {}.", appDefPath); - } - - if (buildInfo.addonDelegate.getAddonMap().size() > 0) { - if (SliderUtils.isUnset(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) { - throw new BadConfigException("addon package can only be specified if main app package is specified."); - } - - List addons = new ArrayList(); - Map addonMap = buildInfo.addonDelegate.getAddonMap(); - for (Map.Entry entry : addonMap.entrySet()) { - String key = entry.getKey(); - String value = entry.getValue(); - if (SliderUtils.isUnset(value)) { - throw new BadConfigException("Invalid path for addon package " + key); - } - File defPath = new File(value); - if (!defPath.exists()) { - throw new BadConfigException("addon folder or package path is not valid."); - } - - Path addonPath = sliderFileSystem.buildAddonDirPath(clustername, key); - String addonPkgName = "addon_" + key + ".zip"; - - log.debug( - "addonMap.get(key): {} addonPath: {} defPath: {} addonPkgName: {}", - addonMap.get(key), addonPath, defPath, addonPkgName); - - appDefinitions.add(new AppDefinition(addonPath, defPath, addonPkgName)); - String addOnKey = AgentKeys.ADDON_PREFIX + key; - Path addonPkgPath = new Path(addonPath, addonPkgName); - log.info("Setting addon package {} to {}.", addOnKey, addonPkgPath); - appConf.getGlobalOptions().set(addOnKey, addonPkgPath); - addons.add(addOnKey); - } - - String existingList = appConf.getGlobalOptions().get(AgentKeys.ADDONS); - if (SliderUtils.isUnset(existingList)) { - existingList = ""; - } - appConf.getGlobalOptions().set(AgentKeys.ADDONS, existingList + StringUtils.join(addons, ",")); - } - } - - - @VisibleForTesting - public List getAppDefinitions() { - return appDefinitions; - } - - // Helper class to hold details for the app and addon packages - static class AppDefinition { - // The target folder where the package will be stored - public Path targetFolderInFs; - // The on disk location of the app def package or folder - public File appDefPkgOrFolder; - // Package name - public String pkgName; - - public AppDefinition(Path targetFolderInFs, File appDefPkgOrFolder, String pkgName) { - this.targetFolderInFs = targetFolderInFs; - this.appDefPkgOrFolder = appDefPkgOrFolder; - this.pkgName = pkgName; - } - - @Override - public String toString() { - return new StringBuilder().append("targetFolderInFs").append(" : ").append(targetFolderInFs.toString()) - .append(", ") - .append("appDefPkgOrFolder").append(" : ").append(appDefPkgOrFolder.toString()) - .append(", ") - .append("pkgName").append(" : ").append(pkgName).toString(); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java index 4f60c069708..8fe2549172d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java @@ -29,6 +29,7 @@ import org.codehaus.jackson.JsonParseException; import org.codehaus.jackson.map.DeserializationConfig; import org.codehaus.jackson.map.JsonMappingException; import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.PropertyNamingStrategy; import org.codehaus.jackson.map.SerializationConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,6 +66,11 @@ public class JsonSerDeser { mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); } + public JsonSerDeser(Class classType, PropertyNamingStrategy namingStrategy) { + this(classType); + mapper.setPropertyNamingStrategy(namingStrategy); + } + /** * Convert from JSON * @param json input diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java index 01444fde4a0..42e103a847f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java @@ -102,19 +102,6 @@ public abstract class AbstractClientProvider extends Configured { } - /** - * Any provider-side alteration of a configuration can take place here. - * @param aggregateConf config to patch - * @throws IOException IO problems - * @throws SliderException Slider-specific issues - */ - public void prepareInstanceConfiguration(AggregateConf aggregateConf) throws - SliderException, - IOException { - //default: do nothing - } - - /** * Prepare the AM settings for launch * @param fileSystem filesystem @@ -234,7 +221,7 @@ public abstract class AbstractClientProvider extends Configured { * @param appDescription brief description of the application * @return */ - public final Set createApplicationTags(String appName, + public static final Set createApplicationTags(String appName, String appVersion, String appDescription) { Set tags = new HashSet<>(); tags.add(SliderUtils.createNameTag(appName)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java deleted file mode 100644 index 41b26e9df9f..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java +++ /dev/null @@ -1,438 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; -import org.apache.hadoop.registry.client.exceptions.InvalidRecordException; -import org.apache.hadoop.registry.client.types.AddressTypes; -import org.apache.hadoop.registry.client.types.Endpoint; -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.service.Service; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.slider.api.ClusterDescription; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.tools.ConfigHelper; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.main.ExitCodeProvider; -import org.apache.slider.server.appmaster.actions.QueueAccess; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; -import org.apache.slider.server.appmaster.state.ContainerReleaseSelector; -import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector; -import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.services.workflow.ForkedProcessService; -import org.apache.slider.server.services.workflow.ServiceParent; -import org.apache.slider.server.services.workflow.WorkflowSequenceService; -import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; - -/** - * The base class for provider services. It lets the implementations - * add sequences of operations, and propagates service failures - * upstream - */ -public abstract class AbstractProviderService - extends WorkflowSequenceService - implements - ProviderCore, - SliderKeys, - ProviderService { - private static final Logger log = - LoggerFactory.getLogger(AbstractProviderService.class); - protected StateAccessForProviders amState; - protected URL amWebAPI; - protected YarnRegistryViewForProviders yarnRegistry; - protected QueueAccess queueAccess; - - protected AbstractProviderService(String name) { - super(name); - setStopIfNoChildServicesAtStartup(false); - } - - @Override - public Configuration getConf() { - return getConfig(); - } - - public StateAccessForProviders getAmState() { - return amState; - } - - public QueueAccess getQueueAccess() { - return queueAccess; - } - - public void setAmState(StateAccessForProviders amState) { - this.amState = amState; - } - - @Override - public String getHumanName() { - return getName().toLowerCase(Locale.ENGLISH); - } - - @Override - public void bind(StateAccessForProviders stateAccessor, - QueueAccess queueAccess, - List liveContainers) { - this.amState = stateAccessor; - this.queueAccess = queueAccess; - } - - @Override - public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) { - this.yarnRegistry = yarnRegistry; - } - - public YarnRegistryViewForProviders getYarnRegistry() { - return yarnRegistry; - } - - @Override - public void notifyContainerCompleted(ContainerId containerId) { - } - - /** - * Load default Configuration - * @param confDir configuration directory - * @return configuration - * @throws BadCommandArgumentsException - * @throws IOException - */ - @Override - public Configuration loadProviderConfigurationInformation(File confDir) - throws BadCommandArgumentsException, IOException { - return new Configuration(false); - } - - /** - * Load a specific XML configuration file for the provider config - * @param confDir configuration directory - * @param siteXMLFilename provider-specific filename - * @return a configuration to be included in status - * @throws BadCommandArgumentsException argument problems - * @throws IOException IO problems - */ - protected Configuration loadProviderConfigurationInformation(File confDir, - String siteXMLFilename) - throws BadCommandArgumentsException, IOException { - Configuration siteConf; - File siteXML = new File(confDir, siteXMLFilename); - if (!siteXML.exists()) { - throw new BadCommandArgumentsException( - "Configuration directory %s doesn't contain %s - listing is %s", - confDir, siteXMLFilename, SliderUtils.listDir(confDir)); - } - - //now read it in - siteConf = ConfigHelper.loadConfFromFile(siteXML); - log.info("{} file is at {}", siteXMLFilename, siteXML); - log.info(ConfigHelper.dumpConfigToString(siteConf)); - return siteConf; - } - - /** - * No-op implementation of this method. - */ - @Override - public void initializeApplicationConfiguration( - AggregateConf instanceDefinition, SliderFileSystem fileSystem, - String roleGroup) - throws IOException, SliderException { - } - - /** - * No-op implementation of this method. - * - * {@inheritDoc} - */ - @Override - public void validateApplicationConfiguration(AggregateConf instance, - File confDir, - boolean secure) - throws IOException, SliderException { - - } - - /** - * Scan through the roles and see if it is supported. - * @param role role to look for - * @return true if the role is known about -and therefore - * that a launcher thread can be deployed to launch it - */ - @Override - public boolean isSupportedRole(String role) { - Collection roles = getRoles(); - for (ProviderRole providedRole : roles) { - if (providedRole.name.equals(role)) { - return true; - } - } - return false; - } - - /** - * override point to allow a process to start executing in this container - * @param instanceDefinition cluster description - * @param confDir configuration directory - * @param env environment - * @param execInProgress the callback for the exec events - * @return false - * @throws IOException - * @throws SliderException - */ - @Override - public boolean exec(AggregateConf instanceDefinition, - File confDir, - Map env, - ProviderCompleted execInProgress) throws IOException, SliderException { - return false; - } - - @SuppressWarnings("ThrowableResultOfMethodCallIgnored") - @Override // ExitCodeProvider - public int getExitCode() { - Throwable cause = getFailureCause(); - if (cause != null) { - //failed for some reason - if (cause instanceof ExitCodeProvider) { - return ((ExitCodeProvider) cause).getExitCode(); - } - } - ForkedProcessService lastProc = latestProcess(); - if (lastProc == null || !lastProc.isProcessTerminated()) { - return 0; - } else { - return lastProc.getExitCode(); - } - } - - /** - * Return the latest forked process service that ran - * @return the forkes service - */ - protected ForkedProcessService latestProcess() { - Service current = getActiveService(); - Service prev = getPreviousService(); - - Service latest = current != null ? current : prev; - if (latest instanceof ForkedProcessService) { - return (ForkedProcessService) latest; - } else { - //its a composite object, so look inside it for a process - if (latest instanceof ServiceParent) { - return getFPSFromParentService((ServiceParent) latest); - } else { - //no match - return null; - } - } - } - - - /** - * Given a parent service, find the one that is a forked process - * @param serviceParent parent - * @return the forked process service or null if there is none - */ - protected ForkedProcessService getFPSFromParentService(ServiceParent serviceParent) { - List services = serviceParent.getServices(); - for (Service s : services) { - if (s instanceof ForkedProcessService) { - return (ForkedProcessService) s; - } - } - return null; - } - - /** - * if we are already running, start this service - */ - protected void maybeStartCommandSequence() { - if (isInState(STATE.STARTED)) { - startNextService(); - } - } - - /** - * Create a new forked process service with the given - * name, environment and command list -then add it as a child - * for execution in the sequence. - * - * @param name command name - * @param env environment - * @param commands command line - * @throws IOException - * @throws SliderException - */ - protected ForkedProcessService queueCommand(String name, - Map env, - List commands) throws - IOException, - SliderException { - ForkedProcessService process = buildProcess(name, env, commands); - //register the service for lifecycle management; when this service - //is terminated, so is the master process - addService(process); - return process; - } - - public ForkedProcessService buildProcess(String name, - Map env, - List commands) throws - IOException, - SliderException { - ForkedProcessService process; - process = new ForkedProcessService(name); - process.init(getConfig()); - process.build(env, commands); - return process; - } - - /* - * Build the provider status, can be empty - * @return the provider status - map of entries to add to the info section - */ - @Override - public Map buildProviderStatus() { - return new HashMap(); - } - - /* - Build the monitor details. The base implementation includes all the external URL endpoints - in the external view - */ - @Override - public Map buildMonitorDetails(ClusterDescription clusterDesc) { - Map details = new LinkedHashMap(); - - // add in all the endpoints - buildEndpointDetails(details); - - return details; - } - - @Override - public void buildEndpointDetails(Map details) { - ServiceRecord self = yarnRegistry.getSelfRegistration(); - - List externals = self.external; - for (Endpoint endpoint : externals) { - String addressType = endpoint.addressType; - if (AddressTypes.ADDRESS_URI.equals(addressType)) { - try { - List urls = RegistryTypeUtils.retrieveAddressURLs(endpoint); - if (!urls.isEmpty()) { - details.put(endpoint.api, new MonitorDetail(urls.get(0).toString(), true)); - } - } catch (InvalidRecordException | MalformedURLException ignored) { - // Ignored - } - - } - - } - } - - @Override - public void applyInitialRegistryDefinitions(URL amWebURI, - ServiceRecord serviceRecord) - throws IOException { - this.amWebAPI = amWebURI; - } - - /** - * {@inheritDoc} - * - * - * @return The base implementation returns the most recent containers first. - */ - @Override - public ContainerReleaseSelector createContainerReleaseSelector() { - return new MostRecentContainerReleaseSelector(); - } - - @Override - public void releaseAssignedContainer(ContainerId containerId) { - // no-op - } - - @Override - public void addContainerRequest(AMRMClient.ContainerRequest req) { - // no-op - } - - @Override - public void cancelSingleRequest(AMRMClient.ContainerRequest request) { - // no-op - } - - @Override - public int cancelContainerRequests(Priority priority1, - Priority priority2, - int count) { - return 0; - } - - @Override - public void updateBlacklist(List blacklistAdditions, - List blacklistRemovals) { - // no-op - } - - @Override - public void execute(List operations) { - for (AbstractRMOperation operation : operations) { - operation.execute(this); - } - } - /** - * No-op implementation of this method. - */ - @Override - public void rebuildContainerDetails(List liveContainers, - String applicationId, Map providerRoles) { - } - - @Override - public boolean processContainerStatus(ContainerId containerId, - ContainerStatus status) { - return false; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java index 97674307515..b07fc29962c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java @@ -19,9 +19,6 @@ package org.apache.slider.providers; import org.apache.hadoop.conf.Configuration; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTree; -import org.apache.slider.core.exceptions.SliderException; import java.util.List; public interface ProviderCore { @@ -31,13 +28,4 @@ public interface ProviderCore { List getRoles(); Configuration getConf(); - - /** - * Verify that an instance definition is considered valid by the provider - * @param instanceDefinition instance definition - * @throws SliderException if the configuration is not valid - */ - void validateInstanceDefinition(AggregateConf instanceDefinition) throws - SliderException; - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java index 761ac0f40e0..e0299e75107 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java @@ -18,7 +18,7 @@ package org.apache.slider.providers; -import org.apache.slider.api.ResourceKeys; +import org.apache.slider.api.resource.Component; /** * Provider role and key for use in app requests. @@ -34,16 +34,8 @@ public final class ProviderRole { public int nodeFailureThreshold; public final long placementTimeoutSeconds; public final String labelExpression; + public final Component component; - public ProviderRole(String name, int id) { - this(name, - name, - id, - PlacementPolicy.DEFAULT, - ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD, - ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS, - ResourceKeys.DEF_YARN_LABEL_EXPRESSION); - } /** * Create a provider role @@ -67,7 +59,7 @@ public final class ProviderRole { policy, nodeFailureThreshold, placementTimeoutSeconds, - labelExpression); + labelExpression, null); } /** @@ -87,7 +79,8 @@ public final class ProviderRole { int policy, int nodeFailureThreshold, long placementTimeoutSeconds, - String labelExpression) { + String labelExpression, + Component component) { this.name = name; if (group == null) { this.group = name; @@ -99,6 +92,8 @@ public final class ProviderRole { this.nodeFailureThreshold = nodeFailureThreshold; this.placementTimeoutSeconds = placementTimeoutSeconds; this.labelExpression = labelExpression; + this.component = component; + } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java index 4ca9326e911..c80de7f69fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java @@ -18,18 +18,15 @@ package org.apache.slider.providers; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.service.Service; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.slider.api.ClusterDescription; +import org.apache.slider.api.resource.Application; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.MapOperations; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.core.main.ExitCodeProvider; @@ -45,128 +42,17 @@ import java.net.URL; import java.util.List; import java.util.Map; -public interface ProviderService extends ProviderCore, - Service, - RMOperationHandlerActions, - ExitCodeProvider { +public interface ProviderService extends Service { /** * Set up the entire container launch context - * @param containerLauncher - * @param instanceDefinition - * @param container - * @param providerRole - * @param sliderFileSystem - * @param generatedConfPath - * @param appComponent - * @param containerTmpDirPath */ void buildContainerLaunchContext(ContainerLauncher containerLauncher, - AggregateConf instanceDefinition, - Container container, - ProviderRole providerRole, - SliderFileSystem sliderFileSystem, - Path generatedConfPath, - MapOperations resourceComponent, - MapOperations appComponent, - Path containerTmpDirPath) throws - IOException, - SliderException; + Application application, Container container, ProviderRole providerRole, + SliderFileSystem sliderFileSystem) throws IOException, SliderException; - /** - * Notify the providers of container completion - * @param containerId container that has completed - */ - void notifyContainerCompleted(ContainerId containerId); - /** - * Execute a process in the AM - * @param instanceDefinition cluster description - * @param confDir configuration directory - * @param env environment - * @param execInProgress the callback for the exec events - * @return true if a process was actually started - * @throws IOException - * @throws SliderException - */ - boolean exec(AggregateConf instanceDefinition, - File confDir, - Map env, - ProviderCompleted execInProgress) throws IOException, - SliderException; - - /** - * Scan through the roles and see if it is supported. - * @param role role to look for - * @return true if the role is known about -and therefore - * that a launcher thread can be deployed to launch it - */ - boolean isSupportedRole(String role); - - /** - * Load a specific XML configuration file for the provider config - * @param confDir configuration directory - * @return a configuration to be included in status - * @throws BadCommandArgumentsException - * @throws IOException - */ - Configuration loadProviderConfigurationInformation(File confDir) - throws BadCommandArgumentsException, IOException; - - /** - * The application configuration should be initialized here - * - * @param instanceDefinition - * @param fileSystem - * @param roleGroup - * @throws IOException - * @throws SliderException - */ - void initializeApplicationConfiguration(AggregateConf instanceDefinition, - SliderFileSystem fileSystem, String roleGroup) throws IOException, - SliderException; - - /** - * This is a validation of the application configuration on the AM. - * Here is where things like the existence of keytabs and other - * not-seen-client-side properties can be tested, before - * the actual process is spawned. - * @param instanceDefinition clusterSpecification - * @param confDir configuration directory - * @param secure flag to indicate that secure mode checks must exist - * @throws IOException IO problemsn - * @throws SliderException any failure - */ - void validateApplicationConfiguration(AggregateConf instanceDefinition, - File confDir, - boolean secure - ) throws IOException, SliderException; - - /* - * Build the provider status, can be empty - * @return the provider status - map of entries to add to the info section - */ - Map buildProviderStatus(); - - /** - * Build a map of data intended for the AM webapp that is specific - * about this provider. The key is some text to be displayed, and the - * value can be a URL that will create an anchor over the key text. - * - * If no anchor is needed/desired, insert the key with a null value. - * @return the details - */ - Map buildMonitorDetails(ClusterDescription clusterSpec); - - /** - * Get a human friendly name for web UIs and messages - * @return a name string. Default is simply the service instance name. - */ - String getHumanName(); - - public void bind(StateAccessForProviders stateAccessor, - QueueAccess queueAccess, - List liveContainers); + void setAMState(StateAccessForProviders stateAccessForProviders); /** * Bind to the YARN registry @@ -174,39 +60,6 @@ public interface ProviderService extends ProviderCore, */ void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry); - /** - * Build up the endpoint details for this service - * @param details - */ - void buildEndpointDetails(Map details); - - /** - * Prior to going live -register the initial service registry data - * @param amWebURI URL to the AM. This may be proxied, so use relative paths - * @param serviceRecord service record to build up - */ - void applyInitialRegistryDefinitions(URL amWebURI, - ServiceRecord serviceRecord) - throws IOException; - - /** - * Create the container release selector for this provider...any policy - * can be implemented - * @return the selector to use for choosing containers. - */ - ContainerReleaseSelector createContainerReleaseSelector(); - - /** - * On AM restart (for whatever reason) this API is required to rebuild the AM - * internal state with the containers which were already assigned and running - * - * @param liveContainers - * @param applicationId - * @param providerRoles - */ - void rebuildContainerDetails(List liveContainers, - String applicationId, Map providerRoles); - /** * Process container status * @return true if status needs to be requested again, false otherwise diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java index f33db9be4fe..f8ec9761c4e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java @@ -18,7 +18,6 @@ package org.apache.slider.providers; -import org.apache.commons.io.FileUtils; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -37,24 +36,21 @@ import org.apache.slider.api.InternalKeys; import org.apache.slider.api.OptionKeys; import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.RoleKeys; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.ConfigFile; +import org.apache.slider.api.resource.Configuration; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.core.registry.docstore.ConfigFormat; -import org.apache.slider.core.registry.docstore.ConfigUtils; -import org.apache.slider.core.registry.docstore.ExportEntry; import org.apache.slider.core.registry.docstore.PublishedConfiguration; import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; -import org.apache.slider.core.registry.docstore.PublishedExports; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.StateAccessForProviders; import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; @@ -66,16 +62,10 @@ import java.io.IOException; import java.net.URI; import java.util.ArrayList; import java.util.Collection; -import java.util.Date; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; import java.util.regex.Pattern; /** @@ -114,7 +104,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys { */ public static boolean addProviderJar( Map providerResources, - Object provider, + Class providerClass, String jarName, SliderFileSystem sliderFileSystem, Path tempPath, @@ -125,7 +115,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys { try { SliderUtils.putJar(providerResources, sliderFileSystem, - provider.getClass(), + providerClass, tempPath, libdir, jarName); @@ -138,32 +128,6 @@ public class ProviderUtils implements RoleKeys, SliderKeys { } } } - - /** - * Add/overwrite the agent tarball (overwritten every time application is - * restarted). - * @param provider an instance of a provider class - * @param tarName name of the tarball to upload - * @param sliderFileSystem the file system - * @param agentDir directory to upload to - * @return true the location could be determined and the file added - * @throws IOException if the upload fails - */ - public static boolean addAgentTar(Object provider, - String tarName, - SliderFileSystem sliderFileSystem, - Path agentDir) throws - IOException { - File localFile = SliderUtils.findContainingJar(provider.getClass()); - if(localFile != null) { - String parentDir = localFile.getParent(); - Path agentTarPath = new Path(parentDir, tarName); - sliderFileSystem.getFileSystem().copyFromLocalFile(false, true, - agentTarPath, agentDir); - return true; - } - return false; - } /** * Loads all dependency jars from the default path. @@ -193,132 +157,24 @@ public class ProviderUtils implements RoleKeys, SliderKeys { libDir, libLocalSrcDir); } - - /** - * Validate the requested number of instances of a component. - *

- * If max <= 0: min <= count - * If max > 0: min <= count <= max - * @param instanceDescription configuration - * @param name node class name - * @param min requested heap size - * @param max maximum value. - * @throws BadCommandArgumentsException if the values are out of range - */ - public void validateNodeCount(AggregateConf instanceDescription, - String name, int min, int max) - throws BadCommandArgumentsException { - MapOperations component = - instanceDescription.getResourceOperations().getComponent(name); - int count; - if (component == null) { - count = 0; - } else { - count = component.getOptionInt(ResourceKeys.COMPONENT_INSTANCES, 0); - } - validateNodeCount(name, count, min, max); - } - - /** - * Validate the count is between min and max. - *

- * If max <= 0: min <= count - * If max > 0: min <= count <= max - * @param name node class name - * @param count requested node count - * @param min requested heap size - * @param max maximum value. - * @throws BadCommandArgumentsException if the values are out of range - */ - public void validateNodeCount(String name, - int count, - int min, - int max) throws BadCommandArgumentsException { - if (count < min) { - throw new BadCommandArgumentsException( - "requested no of %s nodes: %d is below the minimum of %d", name, count, - min); - } - if (max > 0 && count > max) { - throw new BadCommandArgumentsException( - "requested no of %s nodes: %d is above the maximum of %d", name, count, - max); - } - } - - /** - * Copy options beginning with "site.configName." prefix from options map - * to sitexml map, removing the prefix and substituting the tokens - * specified in the tokenMap. - * @param options source map - * @param sitexml destination map - * @param configName optional ".configName" portion of the prefix - * @param tokenMap key/value pairs to substitute into the option values - */ - public void propagateSiteOptions(Map options, - Map sitexml, - String configName, - Map tokenMap) { - String prefix = OptionKeys.SITE_XML_PREFIX + - (!configName.isEmpty() ? configName + "." : ""); - propagateOptions(options, sitexml, tokenMap, prefix); - } - - /** - * Copy options beginning with prefix from options map - * to sitexml map, removing the prefix and substituting the tokens - * specified in the tokenMap. - * @param options source map - * @param sitexml destination map - * @param tokenMap key/value pairs to substitute into the option values - * @param prefix which options to copy to destination map - */ - public void propagateOptions(Map options, - Map sitexml, - Map tokenMap, - String prefix) { - for (Map.Entry entry : options.entrySet()) { - String key = entry.getKey(); - if (key.startsWith(prefix)) { - String envName = key.substring(prefix.length()); - if (!envName.isEmpty()) { - String value = entry.getValue(); - if (tokenMap != null) { - for (Map.Entry token : tokenMap.entrySet()) { - value = value.replaceAll(Pattern.quote(token.getKey()), - token.getValue()); - } - } - sitexml.put(envName, value); - } - } - } - } - - /** - * Substitute tokens into option map values, returning a new map. - * @param options source map - * @param tokenMap key/value pairs to substitute into the option values - * @return map with substituted values - */ - public Map filterSiteOptions(Map options, + // Build key -> value map + // value will be substituted by corresponding data in tokenMap + public Map substituteConfigs(Map configs, Map tokenMap) { - String prefix = OptionKeys.SITE_XML_PREFIX; String format = "${%s}"; Map filteredOptions = new HashMap<>(); - for (Map.Entry entry : options.entrySet()) { + for (Map.Entry entry : configs.entrySet()) { String key = entry.getKey(); - if (key.startsWith(prefix)) { - String value = entry.getValue(); - if (tokenMap != null) { - for (Map.Entry token : tokenMap.entrySet()) { - value = value.replaceAll(Pattern.quote(token.getKey()), - token.getValue()); - } + String value = entry.getValue(); + if (tokenMap != null) { + for (Map.Entry token : tokenMap.entrySet()) { + value = + value.replaceAll(Pattern.quote(token.getKey()), token.getValue()); } - filteredOptions.put(String.format(format, key), value); } + filteredOptions.put(String.format(format, key), value); } + return filteredOptions; } @@ -345,28 +201,27 @@ public class ProviderUtils implements RoleKeys, SliderKeys { return intVal; } + /** * Localize the service keytabs for the application. * @param launcher container launcher - * @param instanceDefinition app specification * @param fileSystem file system - * @param clusterName app name * @throws IOException trouble uploading to HDFS */ public void localizeServiceKeytabs(ContainerLauncher launcher, - AggregateConf instanceDefinition, SliderFileSystem fileSystem, - String clusterName) throws IOException { - ConfTreeOperations appConf = instanceDefinition.getAppConfOperations(); - String keytabPathOnHost = appConf.getComponent(COMPONENT_AM).get( - SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); + SliderFileSystem fileSystem, Application application) throws IOException { + + Configuration conf = application.getConfiguration(); + String keytabPathOnHost = + conf.getProperty(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); if (SliderUtils.isUnset(keytabPathOnHost)) { - String amKeytabName = appConf.getComponent(COMPONENT_AM).get( - SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); - String keytabDir = appConf.getComponent(COMPONENT_AM).get( - SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); + String amKeytabName = + conf.getProperty(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); + String keytabDir = + conf.getProperty(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); // we need to localize the keytab files in the directory Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null, - clusterName); + application.getName()); boolean serviceKeytabsDeployed = false; if (fileSystem.getFileSystem().exists(keytabDirPath)) { FileStatus[] keytabs = fileSystem.getFileSystem().listStatus( @@ -395,591 +250,119 @@ public class ProviderUtils implements RoleKeys, SliderKeys { } - /** - * Upload a local file to the cluster security dir in HDFS. If the file - * already exists, it is not replaced. - * @param resource file to upload - * @param fileSystem file system - * @param clusterName app name - * @return Path of the uploaded file - * @throws IOException file cannot be uploaded - */ - private Path uploadSecurityResource(File resource, - SliderFileSystem fileSystem, String clusterName) throws IOException { - Path certsDir = fileSystem.buildClusterSecurityDirPath(clusterName); - return uploadResource(resource, fileSystem, certsDir); - } - - /** - * Upload a local file to the cluster resources dir in HDFS. If the file - * already exists, it is not replaced. - * @param resource file to upload - * @param fileSystem file system - * @param roleName optional subdirectory (for component-specific resources) - * @param clusterName app name - * @return Path of the uploaded file - * @throws IOException file cannot be uploaded - */ - private Path uploadResource(File resource, SliderFileSystem fileSystem, - String roleName, String clusterName) throws IOException { - Path dir; - if (roleName == null) { - dir = fileSystem.buildClusterResourcePath(clusterName); - } else { - dir = fileSystem.buildClusterResourcePath(clusterName, roleName); - } - return uploadResource(resource, fileSystem, dir); - } - - /** - * Upload a local file to a specified HDFS directory. If the file already - * exists, it is not replaced. - * @param resource file to upload - * @param fileSystem file system - * @param parentDir destination directory in HDFS - * @return Path of the uploaded file - * @throws IOException file cannot be uploaded - */ - private synchronized Path uploadResource(File resource, - SliderFileSystem fileSystem, Path parentDir) throws IOException { - if (!fileSystem.getFileSystem().exists(parentDir)) { - fileSystem.getFileSystem().mkdirs(parentDir, + // 1. Create all config files for a component on hdfs for localization + // 2. Add the config file to localResource + //TODO handle Template format config file + public void createConfigFileAndAddLocalResource(ContainerLauncher launcher, + SliderFileSystem fs, Component component, + Map tokensForSubstitution, + StateAccessForProviders amState) throws IOException { + Path compDir = + new Path(new Path(fs.getAppDir(), "components"), component.getName()); + if (!fs.getFileSystem().exists(compDir)) { + fs.getFileSystem().mkdirs(compDir, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); - } - Path destPath = new Path(parentDir, resource.getName()); - if (!fileSystem.getFileSystem().exists(destPath)) { - FSDataOutputStream os = null; - try { - os = fileSystem.getFileSystem().create(destPath); - byte[] contents = FileUtils.readFileToByteArray(resource); - os.write(contents, 0, contents.length); - os.flush(); - } finally { - IOUtils.closeStream(os); - } - log.info("Uploaded {} to localization path {}", resource, destPath); + log.info("Creating component dir: " + compDir); } else { - log.info("Resource {} already existed at localization path {}", resource, - destPath); - } - - while (!fileSystem.getFileSystem().exists(destPath)) { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - // ignore - } - } - - fileSystem.getFileSystem().setPermission(destPath, - new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE)); - - return destPath; - } - - /** - * Write a configuration property map to a local file in a specified format. - * @param fileSystem file system - * @param file destination file - * @param configFormat file format - * @param configFileDN file description - * @param config properties to save to the file - * @param clusterName app name - * @throws IOException file cannot be created - */ - private synchronized void createConfigFile(SliderFileSystem fileSystem, - File file, ConfigFormat configFormat, String configFileDN, - Map config, String clusterName) throws IOException { - if (file.exists()) { - log.info("Skipping writing {} file {} because it already exists", - configFormat, file); + log.info("Component conf dir already exists: " + compDir); return; } - log.info("Writing {} file {}", configFormat, file); - ConfigUtils.prepConfigForTemplateOutputter(configFormat, config, - fileSystem, clusterName, file.getName()); - PublishedConfiguration publishedConfiguration = - new PublishedConfiguration(configFileDN, - config.entrySet()); - PublishedConfigurationOutputter configurationOutputter = - PublishedConfigurationOutputter.createOutputter(configFormat, - publishedConfiguration); - configurationOutputter.save(file); - } - - /** - * Determine config files requested in the appConf, generate the files, and - * localize them. - * @param launcher container launcher - * @param roleName component name - * @param roleGroup component group - * @param appConf app configurations - * @param configs configurations grouped by config name - * @param env environment variables - * @param fileSystem file system - * @param clusterName app name - * @throws IOException file(s) cannot be uploaded - * @throws BadConfigException file name not specified or file format not - * supported - */ - public void localizeConfigFiles(ContainerLauncher launcher, - String roleName, String roleGroup, - ConfTreeOperations appConf, - Map> configs, - MapOperations env, - SliderFileSystem fileSystem, - String clusterName) - throws IOException, BadConfigException { - for (Entry> configEntry : configs.entrySet()) { - String configFileName = appConf.getComponentOpt(roleGroup, - OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys - .NAME_SUFFIX, null); - String configFileType = appConf.getComponentOpt(roleGroup, - OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys - .TYPE_SUFFIX, null); - if (configFileName == null && configFileType == null) { - // config file not requested, so continue - continue; + for (ConfigFile configFile : component.getConfiguration().getFiles()) { + String fileName = configFile.getSrcFile(); + // substitute file name + for (Map.Entry token : tokensForSubstitution.entrySet()) { + configFile.setDestFile(configFile.getDestFile() + .replaceAll(Pattern.quote(token.getKey()), token.getValue())); } - if (configFileName == null) { - throw new BadConfigException("Config file name null for " + - configEntry.getKey()); - } - if (configFileType == null) { - throw new BadConfigException("Config file type null for " + - configEntry.getKey()); - } - ConfigFormat configFormat = ConfigFormat.resolve(configFileType); - if (configFormat == null) { - throw new BadConfigException("Config format " + configFileType + - " doesn't exist"); - } - boolean perComponent = appConf.getComponentOptBool(roleGroup, - OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys - .PER_COMPONENT, false); - boolean perGroup = appConf.getComponentOptBool(roleGroup, - OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys - .PER_GROUP, false); + // substitute configs + substituteConfigs(configFile.getProps(), tokensForSubstitution); - localizeConfigFile(launcher, roleName, roleGroup, configEntry.getKey(), - configFormat, configFileName, configs, env, fileSystem, - clusterName, perComponent, perGroup); - } - } - - /** - * Create and localize a config file. - * @param launcher container launcher - * @param roleName component name - * @param roleGroup component group - * @param configFileDN config description/name - * @param configFormat config format - * @param configFileName config file name - * @param configs configs grouped by config description/name - * @param env environment variables - * @param fileSystem file system - * @param clusterName app name - * @param perComponent true if file should be created per unique component - * @param perGroup true if file should be created per component group - * @throws IOException file cannot be uploaded - */ - public void localizeConfigFile(ContainerLauncher launcher, - String roleName, String roleGroup, - String configFileDN, ConfigFormat configFormat, String configFileName, - Map> configs, - MapOperations env, - SliderFileSystem fileSystem, - String clusterName, - boolean perComponent, - boolean perGroup) - throws IOException { - if (launcher == null) { - return; - } - Map config = ConfigUtils.replacePropsInConfig( - configs.get(configFileDN), env.options); - String fileName = ConfigUtils.replaceProps(config, configFileName); - File localFile = new File(RESOURCE_DIR); - if (!localFile.exists()) { - if (!localFile.mkdir() && !localFile.exists()) { - throw new IOException(RESOURCE_DIR + " could not be created!"); - } - } - - String folder = null; - if (perComponent) { - folder = roleName; - } else if (perGroup) { - folder = roleGroup; - } - if (folder != null) { - localFile = new File(localFile, folder); - if (!localFile.exists()) { - if (!localFile.mkdir() && !localFile.exists()) { - throw new IOException(localFile + " could not be created!"); - } - } - } - localFile = new File(localFile, new File(fileName).getName()); - - log.info("Localizing {} configs to config file {} (destination {}) " + - "based on {} configs", config.size(), localFile, fileName, - configFileDN); - if (!localFile.exists()) { - createConfigFile(fileSystem, localFile, configFormat, configFileDN, - config, clusterName); - } else { - log.info("Local {} file {} already exists", configFormat, localFile); - } - Path destPath = uploadResource(localFile, fileSystem, folder, clusterName); - LocalResource configResource = fileSystem.createAmResource(destPath, - LocalResourceType.FILE); - - File destFile = new File(fileName); - if (destFile.isAbsolute()) { - launcher.addLocalResource( - RESOURCE_DIR + "/" + destFile.getName(), - configResource, fileName); - } else { - launcher.addLocalResource(APP_CONF_DIR + "/" + fileName, - configResource); - } - } - - /** - * Localize application tarballs and other resources requested by the app. - * @param launcher container launcher - * @param fileSystem file system - * @param appConf app configurations - * @param roleGroup component group - * @param clusterName app name - * @throws IOException resources cannot be uploaded - * @throws BadConfigException package name or type is not specified - */ - public void localizePackages(ContainerLauncher launcher, - SliderFileSystem fileSystem, ConfTreeOperations appConf, String roleGroup, - String clusterName) throws IOException, BadConfigException { - for (Entry> pkg : - getPackages(roleGroup, appConf).entrySet()) { - String pkgName = pkg.getValue().get(OptionKeys.NAME_SUFFIX); - String pkgType = pkg.getValue().get(OptionKeys.TYPE_SUFFIX); - Path pkgPath = fileSystem.buildResourcePath(pkgName); - if (!fileSystem.isFile(pkgPath)) { - pkgPath = fileSystem.buildResourcePath(clusterName, - pkgName); - } - if (!fileSystem.isFile(pkgPath)) { - throw new IOException("Package doesn't exist as a resource: " + - pkgName); - } - log.info("Adding resource {}", pkgName); - LocalResourceType type = LocalResourceType.FILE; - if ("archive".equals(pkgType)) { - type = LocalResourceType.ARCHIVE; - } - LocalResource packageResource = fileSystem.createAmResource( - pkgPath, type); - launcher.addLocalResource(APP_PACKAGES_DIR, packageResource); - } - } - - /** - * Build a map of configuration description/name to configuration key/value - * properties, with all known tokens substituted into the property values. - * @param appConf app configurations - * @param internalsConf internal configurations - * @param containerId container ID - * @param roleName component name - * @param roleGroup component group - * @param amState access to AM state - * @return configuration properties grouped by config description/name - */ - public Map> buildConfigurations( - ConfTreeOperations appConf, ConfTreeOperations internalsConf, - String containerId, String clusterName, String roleName, String roleGroup, - StateAccessForProviders amState) { - - Map> configurations = new TreeMap<>(); - Map tokens = getStandardTokenMap(appConf, - internalsConf, roleName, roleGroup, containerId, clusterName); - - Set configs = new HashSet<>(); - configs.addAll(getApplicationConfigurationTypes(roleGroup, appConf)); - configs.addAll(getSystemConfigurationsRequested(appConf)); - - for (String configType : configs) { - addNamedConfiguration(configType, appConf.getGlobalOptions().options, - configurations, tokens, amState); - if (appConf.getComponent(roleGroup) != null) { - addNamedConfiguration(configType, - appConf.getComponent(roleGroup).options, configurations, tokens, - amState); - } - } - - //do a final replacement of re-used configs - dereferenceAllConfigs(configurations); - - return configurations; - } - - /** - * Substitute "site." prefixed configuration values into other configuration - * values where needed. The format for these substitutions is that - * {@literal ${@//site/configDN/key}} will be replaced by the value for the - * "site.configDN.key" property. - * @param configurations configuration properties grouped by config - * description/name - */ - public void dereferenceAllConfigs( - Map> configurations) { - Map allConfigs = new HashMap<>(); - String lookupFormat = "${@//site/%s/%s}"; - for (Map.Entry> entry : configurations.entrySet()) { - Map configBucket = entry.getValue(); - for(Map.Entry config: configBucket.entrySet()) { - allConfigs.put(String.format(lookupFormat, entry.getKey(), config.getKey()), - config.getValue()); - } - } - - boolean finished = false; - while (!finished) { - finished = true; - for (Map.Entry entry : allConfigs.entrySet()) { - String configValue = entry.getValue(); - for (Map.Entry lookUpEntry : allConfigs.entrySet()) { - String lookUpValue = lookUpEntry.getValue(); - if (lookUpValue.contains("${@//site/")) { - continue; - } - String lookUpKey = lookUpEntry.getKey(); - if (configValue != null && configValue.contains(lookUpKey)) { - configValue = configValue.replace(lookUpKey, lookUpValue); + // write configs onto hdfs + PublishedConfiguration publishedConfiguration = + new PublishedConfiguration(fileName, + configFile.getProps().entrySet()); + Path remoteFile = new Path(compDir, fileName); + if (!fs.getFileSystem().exists(remoteFile)) { + synchronized (this) { + if (!fs.getFileSystem().exists(remoteFile)) { + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter( + ConfigFormat.resolve(configFile.getType().toString()), + publishedConfiguration); + FSDataOutputStream os = null; + try { + os = fs.getFileSystem().create(remoteFile); + configurationOutputter.save(os); + os.flush(); + log.info("Created config file on hdfs: " + remoteFile); + } finally { + IOUtils.closeStream(os); + } } } - if (configValue != null && !configValue.equals(entry.getValue())) { - finished = false; - allConfigs.put(entry.getKey(), configValue); - } + } + + // Publish configs + amState.getPublishedSliderConfigurations() + .put(configFile.getSrcFile(), publishedConfiguration); + + // Add resource for localization + LocalResource configResource = + fs.createAmResource(remoteFile, LocalResourceType.FILE); + File destFile = new File(configFile.getDestFile()); + //TODO why to we need to differetiate RESOURCE_DIR vs APP_CONF_DIR + if (destFile.isAbsolute()) { + String symlink = RESOURCE_DIR + "/" + fileName; + launcher.addLocalResource(symlink, configResource, + configFile.getDestFile()); + log.info("Add config file for localization: " + symlink + " -> " + + configResource.getResource().getFile() + ", dest mount path: " + + configFile.getDestFile()); + } else { + String symlink = APP_CONF_DIR + "/" + fileName; + launcher.addLocalResource(symlink, configResource); + log.info("Add config file for localization: " + symlink + " -> " + + configResource.getResource().getFile()); } } - for (Map.Entry> configEntry : configurations - .entrySet()) { - Map configBucket = configEntry.getValue(); - for (Map.Entry entry: configBucket.entrySet()) { - String configName = entry.getKey(); - String configValue = entry.getValue(); - for (Map.Entry lookUpEntry : allConfigs.entrySet()) { - String lookUpValue = lookUpEntry.getValue(); - if (lookUpValue.contains("${@//site/")) { - continue; - } - String lookUpKey = lookUpEntry.getKey(); - if (configValue != null && configValue.contains(lookUpKey)) { - configValue = configValue.replace(lookUpKey, lookUpValue); - } - } - configBucket.put(configName, configValue); - } - } - } - - /** - * Return a set of configuration description/names represented in the app. - * configuration - * @param roleGroup component group - * @param appConf app configurations - * @return set of configuration description/names - */ - public Set getApplicationConfigurationTypes(String roleGroup, - ConfTreeOperations appConf) { - Set configList = new HashSet<>(); - - String prefix = OptionKeys.CONF_FILE_PREFIX; - String suffix = OptionKeys.TYPE_SUFFIX; - MapOperations component = appConf.getComponent(roleGroup); - if (component != null) { - addConfsToList(component, configList, prefix, suffix); - } - addConfsToList(appConf.getGlobalOptions(), configList, prefix, suffix); - - return configList; - } - - /** - * Finds all configuration description/names of the form - * prefixconfigDNsuffix in the configuration (e.g. conf.configDN.type). - * @param confMap configuration properties - * @param confList set containing configuration description/names - * @param prefix configuration key prefix to match - * @param suffix configuration key suffix to match - */ - private void addConfsToList(Map confMap, - Set confList, String prefix, String suffix) { - for (Entry entry : confMap.entrySet()) { - String key = entry.getKey(); - if (key.startsWith(prefix) && key.endsWith(suffix)) { - String confName = key.substring(prefix.length(), - key.length() - suffix.length()); - if (!confName.isEmpty()) { - confList.add(confName); - } - } - } - } - - /** - * Build a map of package description/name to package key/value properties - * (there should be two properties, type and name). - * @param roleGroup component group - * @param appConf app configurations - * @return map of package description/name to package key/value properties - * @throws BadConfigException package name or type is not specified - */ - public Map> getPackages(String roleGroup, - ConfTreeOperations appConf) throws BadConfigException { - Map> packages = new HashMap<>(); - String prefix = OptionKeys.PKG_FILE_PREFIX; - String typeSuffix = OptionKeys.TYPE_SUFFIX; - String nameSuffix = OptionKeys.NAME_SUFFIX; - MapOperations component = appConf.getComponent(roleGroup); - if (component == null) { - component = appConf.getGlobalOptions(); - } - for (Map.Entry entry : component.entrySet()) { - String key = entry.getKey(); - if (key.startsWith(prefix)) { - String confName; - String type; - if (key.endsWith(typeSuffix)) { - confName = key.substring(prefix.length(), key.length() - typeSuffix.length()); - type = typeSuffix; - } else if (key.endsWith(nameSuffix)) { - confName = key.substring(prefix.length(), key.length() - nameSuffix.length()); - type = nameSuffix; - } else { - continue; - } - if (!packages.containsKey(confName)) { - packages.put(confName, new HashMap()); - } - packages.get(confName).put(type, entry.getValue()); - } - } - - for (Entry> pkg : packages.entrySet()) { - if (!pkg.getValue().containsKey(OptionKeys.TYPE_SUFFIX)) { - throw new BadConfigException("Package " + pkg.getKey() + " doesn't " + - "have a package type"); - } - if (!pkg.getValue().containsKey(OptionKeys.NAME_SUFFIX)) { - throw new BadConfigException("Package " + pkg.getKey() + " doesn't " + - "have a package name"); - } - } - - return packages; - } - - /** - * Return system configurations requested by the app. - * @param appConf app configurations - * @return set of system configurations - */ - public Set getSystemConfigurationsRequested( - ConfTreeOperations appConf) { - Set configList = new HashSet<>(); - - String configTypes = appConf.get(SYSTEM_CONFIGS); - if (configTypes != null && configTypes.length() > 0) { - String[] configs = configTypes.split(","); - for (String config : configs) { - configList.add(config.trim()); - } - } - - return configList; - } - - /** - * For a given config description/name, pull out its site configs from the - * source config map, remove the site.configDN. prefix from them, and place - * them into a new config map using the {@link #propagateSiteOptions} method - * (with tokens substituted). This new k/v map is put as the value for the - * configDN key in the configurations map. - * @param configName config description/name - * @param sourceConfig config containing site.* properties - * @param configurations configuration map to be populated - * @param tokens initial substitution tokens - * @param amState access to AM state - */ - private void addNamedConfiguration(String configName, - Map sourceConfig, - Map> configurations, - Map tokens, StateAccessForProviders amState) { - Map config = new HashMap<>(); - if (configName.equals(GLOBAL_CONFIG_TAG)) { - addDefaultGlobalConfig(config); - } - // add role hosts to tokens - addRoleRelatedTokens(tokens, amState); - propagateSiteOptions(sourceConfig, config, configName, tokens); - - configurations.put(configName, config); } /** * Get initial token map to be substituted into config values. * @param appConf app configurations - * @param internals internal configurations - * @param componentName component name - * @param componentGroup component group - * @param clusterName app name - * @return tokens to replace - */ - public Map getStandardTokenMap(ConfTreeOperations appConf, - ConfTreeOperations internals, String componentName, - String componentGroup, String clusterName) { - return getStandardTokenMap(appConf, internals, componentName, - componentGroup, null, clusterName); - } - - /** - * Get initial token map to be substituted into config values. - * @param appConf app configurations - * @param internals internal configurations * @param componentName component name * @param componentGroup component group * @param containerId container ID * @param clusterName app name * @return tokens to replace */ - public Map getStandardTokenMap(ConfTreeOperations appConf, - ConfTreeOperations internals, String componentName, + public Map getStandardTokenMap( + Configuration appConf, Configuration componentConf, String componentName, String componentGroup, String containerId, String clusterName) { Map tokens = new HashMap<>(); if (containerId != null) { tokens.put("${CONTAINER_ID}", containerId); } - String nnuri = appConf.get("site.fs.defaultFS"); - tokens.put("${NN_URI}", nnuri); - tokens.put("${NN_HOST}", URI.create(nnuri).getHost()); - tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS)); - tokens.put("${DEFAULT_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH)); - String prefix = appConf.getComponentOpt(componentGroup, ROLE_PREFIX, - null); + String nnuri = appConf.getProperty("fs.defaultFS"); + if (nnuri != null && !nnuri.isEmpty()) { + tokens.put("${NN_URI}", nnuri); + tokens.put("${NN_HOST}", URI.create(nnuri).getHost()); + } + tokens.put("${ZK_HOST}", appConf.getProperty(OptionKeys.ZOOKEEPER_HOSTS)); + tokens.put("${DEFAULT_ZK_PATH}", appConf.getProperty(OptionKeys.ZOOKEEPER_PATH)); + String prefix = componentConf.getProperty(ROLE_PREFIX); String dataDirSuffix = ""; if (prefix == null) { prefix = ""; } else { dataDirSuffix = "_" + SliderUtils.trimPrefix(prefix); } - tokens.put("${DEFAULT_DATA_DIR}", internals.getGlobalOptions() - .getOption(InternalKeys.INTERNAL_DATA_DIR_PATH, null) + dataDirSuffix); - tokens.put("${JAVA_HOME}", appConf.get(JAVA_HOME)); + tokens.put("${DEFAULT_DATA_DIR}", + appConf.getProperty(InternalKeys.INTERNAL_DATA_DIR_PATH) + + dataDirSuffix); + tokens.put("${JAVA_HOME}", appConf.getProperty(JAVA_HOME)); tokens.put("${COMPONENT_NAME}", componentName); tokens.put("${COMPONENT_NAME.lc}", componentName.toLowerCase()); tokens.put("${COMPONENT_PREFIX}", prefix); @@ -1005,7 +388,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys { * @param tokens existing tokens * @param amState access to AM state */ - public void addRoleRelatedTokens(Map tokens, + public void addRoleHostTokens(Map tokens, StateAccessForProviders amState) { if (amState == null) { return; @@ -1019,26 +402,6 @@ public class ProviderUtils implements RoleKeys, SliderKeys { } } - /** - * Add global configuration properties. - * @param config map where default global properties will be added - */ - private void addDefaultGlobalConfig(Map config) { - config.put("app_log_dir", "${LOG_DIR}"); - config.put("app_pid_dir", "${WORK_DIR}/app/run"); - config.put("app_install_dir", "${WORK_DIR}/app/install"); - config.put("app_conf_dir", "${WORK_DIR}/" + APP_CONF_DIR); - config.put("app_input_conf_dir", "${WORK_DIR}/" + PROPAGATED_CONF_DIR_NAME); - - // add optional parameters only if they are not already provided - if (!config.containsKey("pid_file")) { - config.put("pid_file", "${WORK_DIR}/app/run/component.pid"); - } - if (!config.containsKey("app_root")) { - config.put("app_root", "${WORK_DIR}/app/install"); - } - } - /** * Return a list of hosts based on current ClusterNodes. * @param values cluster nodes @@ -1101,82 +464,4 @@ public class ProviderUtils implements RoleKeys, SliderKeys { containerId, e); } } - - /** - * Publish a named property bag that may contain name-value pairs for app - * configurations such as hbase-site. - * @param name config file identifying name - * @param description config file description - * @param entries config file properties - * @param amState access to AM state - */ - public void publishApplicationInstanceData(String name, String description, - Iterable> entries, - StateAccessForProviders amState) { - PublishedConfiguration pubconf = new PublishedConfiguration(description, - entries); - log.info("publishing {}", pubconf); - amState.getPublishedSliderConfigurations().put(name, pubconf); - } - - /** - * Publish an export group. - * @param exportGroup export groups - * @param amState access to AM state - * @param groupName export group name - */ - public void publishExportGroup( - Map> exportGroup, - StateAccessForProviders amState, String groupName) { - // Publish in old format for the time being - Map simpleEntries = new HashMap<>(); - for (Entry> entry : exportGroup.entrySet()) { - Set exports = entry.getValue(); - if (SliderUtils.isNotEmpty(exports)) { - Set values = new TreeSet<>(); - for (ExportEntry export : exports) { - values.add(export.getValue()); - } - simpleEntries.put(entry.getKey(), StringUtils.join(",", values)); - } - } - publishApplicationInstanceData(groupName, groupName, - simpleEntries.entrySet(), amState); - - PublishedExports exports = new PublishedExports(groupName); - exports.setUpdated(new Date().getTime()); - exports.putValues(exportGroup.entrySet()); - amState.getPublishedExportsSet().put(groupName, exports); - } - - public Map getExports(ConfTreeOperations appConf, - String roleGroup) { - Map exports = new HashMap<>(); - propagateOptions(appConf.getComponent(roleGroup).options, exports, - null, OptionKeys.EXPORT_PREFIX); - return exports; - } - - public String getGroupKey(String roleGroup, ConfTreeOperations appConf) { - String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, ""); - return getNameOrGroupKey(rolePrefix, roleGroup); - } - - public String getNameKey(String roleName, String roleGroup, - ConfTreeOperations appConf) { - String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, ""); - return getNameOrGroupKey(rolePrefix, roleName); - } - - public String getNameOrGroupKey(String rolePrefix, String roleNameOrGroup) { - if (!rolePrefix.isEmpty()) { - if (!roleNameOrGroup.startsWith(rolePrefix)) { - log.warn("Something went wrong, {} doesn't start with {}", - roleNameOrGroup, rolePrefix); - return null; - } - roleNameOrGroup = roleNameOrGroup.substring(rolePrefix.length()); - } - return roleNameOrGroup.toUpperCase(Locale.ENGLISH); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java index 86d87acb601..8b88c2895bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java @@ -22,7 +22,6 @@ import org.apache.slider.common.SliderKeys; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.providers.AbstractClientProvider; import org.apache.slider.providers.ProviderRole; @@ -30,14 +29,10 @@ import org.apache.slider.providers.ProviderUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Set; -import static org.apache.slider.providers.docker.DockerKeys.DOCKER_IMAGE; - public class DockerClientProvider extends AbstractClientProvider implements SliderKeys { @@ -64,35 +59,7 @@ public class DockerClientProvider extends AbstractClientProvider public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws SliderException { super.validateInstanceDefinition(instanceDefinition, fs); - - ConfTreeOperations appConf = instanceDefinition.getAppConfOperations(); - ConfTreeOperations resources = instanceDefinition.getResourceOperations(); - - for (String roleGroup : resources.getComponentNames()) { - if (roleGroup.equals(COMPONENT_AM)) { - continue; - } - if (appConf.getComponentOpt(roleGroup, DOCKER_IMAGE, null) == null && - appConf.getGlobalOptions().get(DOCKER_IMAGE) == null) { - throw new BadConfigException("Property " + DOCKER_IMAGE + " not " + - "specified for " + roleGroup); - } - - providerUtils.getPackages(roleGroup, appConf); - - if (appConf.getComponentOptBool(roleGroup, AM_CONFIG_GENERATION, false)) { - // build and localize configuration files - Map> configurations = - providerUtils.buildConfigurations(appConf, appConf, null, - null, roleGroup, roleGroup, null); - try { - providerUtils.localizeConfigFiles(null, roleGroup, roleGroup, appConf, - configurations, null, fs, null); - } catch (IOException e) { - throw new BadConfigException(e.toString()); - } - } - } + //TODO validate Application payload, part of that is already done in ApplicationApiService, need to do more } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java index 63416cc8249..511f7bc8aea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java @@ -17,286 +17,129 @@ */ package org.apache.slider.providers.docker; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.slider.api.ClusterDescription; -import org.apache.slider.api.ClusterNode; -import org.apache.slider.api.OptionKeys; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.ContainerState; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.exceptions.SliderException; import org.apache.slider.core.launch.CommandLineBuilder; import org.apache.slider.core.launch.ContainerLauncher; -import org.apache.slider.core.registry.docstore.ConfigFormat; -import org.apache.slider.core.registry.docstore.ConfigUtils; -import org.apache.slider.core.registry.docstore.ExportEntry; -import org.apache.slider.providers.AbstractProviderService; -import org.apache.slider.providers.MonitorDetail; -import org.apache.slider.providers.ProviderCore; +import org.apache.slider.core.registry.docstore.PublishedConfiguration; import org.apache.slider.providers.ProviderRole; +import org.apache.slider.providers.ProviderService; import org.apache.slider.providers.ProviderUtils; import org.apache.slider.server.appmaster.state.RoleInstance; +import org.apache.slider.server.appmaster.state.StateAccessForProviders; +import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; -import java.net.URL; -import java.util.Collections; -import java.util.Date; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Scanner; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.regex.Pattern; -public class DockerProviderService extends AbstractProviderService implements - ProviderCore, - DockerKeys, - SliderKeys { +public class DockerProviderService extends AbstractService + implements ProviderService, DockerKeys, SliderKeys { protected static final Logger log = LoggerFactory.getLogger(DockerProviderService.class); private static final ProviderUtils providerUtils = new ProviderUtils(log); - private static final String EXPORT_GROUP = "quicklinks"; - private static final String APPLICATION_TAG = "application"; - private static final String HOST_KEY_FORMAT = "${%s_HOST}"; - private static final String IP_KEY_FORMAT = "${%s_IP}"; - private static final String VARIABLE_INDICATOR = "${"; - - private String clusterName = null; - private SliderFileSystem fileSystem = null; - - private final Map> exportMap = - new ConcurrentHashMap<>(); + private static final String QUICK_LINKS = "quicklinks"; + protected StateAccessForProviders amState; + protected YarnRegistryViewForProviders yarnRegistry; protected DockerProviderService() { super("DockerProviderService"); } @Override - public List getRoles() { - return Collections.emptyList(); + public void setAMState(StateAccessForProviders stateAccessor) { + this.amState = stateAccessor; } @Override - public boolean isSupportedRole(String role) { - return true; + public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) { + this.yarnRegistry = yarnRegistry; } - @Override - public void validateInstanceDefinition(AggregateConf instanceDefinition) - throws SliderException { - } - - private String getClusterName() { - if (SliderUtils.isUnset(clusterName)) { - clusterName = getAmState().getInternalsSnapshot().get(OptionKeys.APPLICATION_NAME); - } - return clusterName; - } - - @Override public void buildContainerLaunchContext(ContainerLauncher launcher, - AggregateConf instanceDefinition, Container container, - ProviderRole providerRole, SliderFileSystem fileSystem, - Path generatedConfPath, MapOperations resourceComponent, - MapOperations appComponent, Path containerTmpDirPath) + Application application, Container container, ProviderRole providerRole, + SliderFileSystem fileSystem) throws IOException, SliderException { String roleName = providerRole.name; String roleGroup = providerRole.group; - log.info("Build launch context for Docker"); - log.debug(instanceDefinition.toString()); - - ConfTreeOperations appConf = instanceDefinition.getAppConfOperations(); + Component component = providerRole.component; launcher.setYarnDockerMode(true); - launcher.setDockerImage(appConf.getComponentOpt(roleGroup, DOCKER_IMAGE, - null)); - launcher.setDockerNetwork(appConf.getComponentOpt(roleGroup, DOCKER_NETWORK, - DEFAULT_DOCKER_NETWORK)); - launcher.setRunPrivilegedContainer(appConf.getComponentOptBool(roleGroup, - DOCKER_USE_PRIVILEGED, DEFAULT_DOCKER_USE_PRIVILEGED)); + launcher.setDockerImage(component.getArtifact().getId()); + launcher.setDockerNetwork(component.getConfiguration() + .getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK)); + launcher.setRunPrivilegedContainer(component.getRunPrivilegedContainer()); - // Set the environment - Map standardTokens = providerUtils.getStandardTokenMap( - getAmState().getAppConfSnapshot(), getAmState().getInternalsSnapshot(), - roleName, roleGroup, container.getId().toString(), getClusterName()); - Map replaceTokens = providerUtils.filterSiteOptions( - appConf.getComponent(roleGroup).options, standardTokens); - replaceTokens.putAll(standardTokens); - launcher.putEnv(SliderUtils.buildEnvMap(appComponent, replaceTokens)); + // Generate tokens (key-value pair) for config substitution. + Map standardTokens = providerUtils + .getStandardTokenMap(application.getConfiguration(), + component.getConfiguration(), roleName, roleGroup, + container.getId().toString(), application.getName()); + Map tokensForSubstitution = providerUtils.substituteConfigs( + component.getConfiguration().getProperties(), standardTokens); - String workDir = ApplicationConstants.Environment.PWD.$(); - launcher.setEnv("WORK_DIR", workDir); - log.info("WORK_DIR set to {}", workDir); - String logDir = ApplicationConstants.LOG_DIR_EXPANSION_VAR; - launcher.setEnv("LOG_DIR", logDir); - log.info("LOG_DIR set to {}", logDir); + tokensForSubstitution.putAll(standardTokens); + + // Set the environment variables + launcher.putEnv(SliderUtils + .buildEnvMap(component.getConfiguration(), tokensForSubstitution)); + launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$()); + launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR); if (System.getenv(HADOOP_USER_NAME) != null) { launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME)); } - //add english env launcher.setEnv("LANG", "en_US.UTF-8"); launcher.setEnv("LC_ALL", "en_US.UTF-8"); launcher.setEnv("LANGUAGE", "en_US.UTF-8"); - //local resources - providerUtils.localizePackages(launcher, fileSystem, appConf, roleGroup, - getClusterName()); + for (Entry entry : launcher.getEnv().entrySet()) { + tokensForSubstitution.put("${" + entry.getKey() + "}", entry.getValue()); + } + + providerUtils.addRoleHostTokens(tokensForSubstitution, amState); + + log.info("Token for substitution: " + tokensForSubstitution); if (SliderUtils.isHadoopClusterSecure(getConfig())) { - providerUtils.localizeServiceKeytabs(launcher, instanceDefinition, - fileSystem, getClusterName()); + //TODO localize key tabs, WHY is this code needed ? WHY DOES CONTAINER REQUIRE AM KEYTAB?? + providerUtils.localizeServiceKeytabs(launcher, fileSystem, application); } - if (appComponent.getOptionBool(AM_CONFIG_GENERATION, false)) { - // build and localize configuration files - Map> configurations = - providerUtils.buildConfigurations( - instanceDefinition.getAppConfOperations(), - instanceDefinition.getInternalOperations(), - container.getId().toString(), getClusterName(), - roleName, roleGroup, getAmState()); - providerUtils.localizeConfigFiles(launcher, roleName, roleGroup, - appConf, configurations, launcher.getEnv(), fileSystem, - getClusterName()); - } - - //add the configuration resources - launcher.addLocalResources(fileSystem.submitDirectory( - generatedConfPath, - PROPAGATED_CONF_DIR_NAME)); + // create config file on hdfs and add local resource + providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem, + component, tokensForSubstitution, amState); CommandLineBuilder operation = new CommandLineBuilder(); - operation.add(appConf.getComponentOpt(roleGroup, DOCKER_START_COMMAND, - "/bin/bash")); - + operation.add(component.getLaunchCommand()); operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + OUT_FILE + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + ERR_FILE); - launcher.addCommand(operation.build()); - // Additional files to localize - String appResourcesString = instanceDefinition.getAppConfOperations() - .getGlobalOptions().getOption(APP_RESOURCES, null); - log.info("Configuration value for extra resources to localize: {}", appResourcesString); - if (null != appResourcesString) { - try (Scanner scanner = new Scanner(appResourcesString).useDelimiter(",")) { - while (scanner.hasNext()) { - String resource = scanner.next(); - Path resourcePath = new Path(resource); - LocalResource extraResource = fileSystem.createAmResource( - fileSystem.getFileSystem().resolvePath(resourcePath), - LocalResourceType.FILE); - String destination = APP_RESOURCES_DIR + "/" + resourcePath.getName(); - log.info("Localizing {} to {}", resourcePath, destination); - // TODO Can we try harder to avoid collisions? - launcher.addLocalResource(destination, extraResource); - } - } - } + // publish exports + // TODO move this to app level, no need to do this for every container launch + providerUtils + .substituteConfigs(application.getQuicklinks(), tokensForSubstitution); + PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS, + application.getQuicklinks().entrySet()); + amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf); } - @Override - public void initializeApplicationConfiguration( - AggregateConf instanceDefinition, SliderFileSystem fileSystem, - String roleGroup) - throws IOException, SliderException { - this.fileSystem = fileSystem; - } - - @Override - public void applyInitialRegistryDefinitions(URL amWebURI, - ServiceRecord serviceRecord) - throws IOException { - super.applyInitialRegistryDefinitions(amWebURI, serviceRecord); - - // identify client component - String clientName = null; - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - for (String component : appConf.getComponentNames()) { - if (COMPONENT_TYPE_CLIENT.equals(appConf.getComponentOpt(component, - COMPONENT_TYPE_KEY, null))) { - clientName = component; - break; - } - } - if (clientName == null) { - log.info("No client component specified, not publishing client configs"); - return; - } - - // register AM-generated client configs - // appConf should already be resolved! - MapOperations clientOperations = appConf.getComponent(clientName); - if (!clientOperations.getOptionBool(AM_CONFIG_GENERATION, false)) { - log.info("AM config generation is false, not publishing client configs"); - return; - } - - // build and localize configuration files - Map> configurations = - providerUtils.buildConfigurations(appConf, getAmState() - .getInternalsSnapshot(), null, getClusterName(), clientName, - clientName, getAmState()); - - for (Map.Entry> entry : configurations.entrySet()) { - String configFileDN = entry.getKey(); - String configFileName = appConf.getComponentOpt(clientName, - OptionKeys.CONF_FILE_PREFIX + configFileDN + OptionKeys - .NAME_SUFFIX, null); - String configFileType = appConf.getComponentOpt(clientName, - OptionKeys.CONF_FILE_PREFIX + configFileDN + OptionKeys - .TYPE_SUFFIX, null); - if (configFileName == null || configFileType == null) { - continue; - } - ConfigFormat configFormat = ConfigFormat.resolve(configFileType); - - Map config = entry.getValue(); - ConfigUtils.prepConfigForTemplateOutputter(configFormat, config, - fileSystem, getClusterName(), - new File(configFileName).getName()); - providerUtils.publishApplicationInstanceData(configFileDN, configFileDN, - config.entrySet(), getAmState()); - } - } - - @Override - public void notifyContainerCompleted(ContainerId containerId) { - if (containerId != null) { - String containerIdStr = containerId.toString(); - log.info("Removing container exports for {}", containerIdStr); - for (Set exportEntries : exportMap.values()) { - for (Iterator iter = exportEntries.iterator(); - iter.hasNext();) { - ExportEntry entry = iter.next(); - if (containerIdStr.equals(entry.getContainerId())) { - iter.remove(); - } - } - } - } - } - - @Override public boolean processContainerStatus(ContainerId containerId, ContainerStatus status) { log.debug("Handling container status: {}", status); @@ -304,144 +147,24 @@ public class DockerProviderService extends AbstractProviderService implements SliderUtils.isUnset(status.getHost())) { return true; } - RoleInstance instance = getAmState().getOwnedContainer(containerId); + RoleInstance instance = amState.getOwnedContainer(containerId); if (instance == null) { // container is completed? return false; } - String roleName = instance.role; - String roleGroup = instance.group; - String containerIdStr = containerId.toString(); - - providerUtils.updateServiceRecord(getAmState(), yarnRegistry, - containerIdStr, roleName, status.getIPs(), status.getHost()); - - publishExportGroups(containerIdStr, roleName, roleGroup, - status.getHost(), status.getIPs()); + providerUtils.updateServiceRecord(amState, yarnRegistry, + containerId.toString(), instance.role, status.getIPs(), status.getHost()); + // TODO publish ip and host + org.apache.slider.api.resource.Container container = + instance.providerRole.component.getContainer(containerId.toString()); + if (container != null) { + container.setIp(StringUtils.join(",", status.getIPs())); + container.setHostname(status.getHost()); + container.setState(ContainerState.READY); + } else { + log.warn(containerId + " not found in Application!"); + } return false; } - - /** - * This method looks for configuration properties of the form - * export.key,value and publishes the key,value pair. Standard tokens are - * substituted into the value, and COMPONENTNAME_HOST and THIS_HOST tokens - * are substituted with the actual hostnames of the containers. - */ - protected void publishExportGroups(String containerId, - String roleName, String roleGroup, String thisHost, List ips) { - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot(); - - Map exports = providerUtils.getExports( - getAmState().getAppConfSnapshot(), roleGroup); - - // publish export groups if any - Map standardTokens = providerUtils.getStandardTokenMap( - appConf, internalsConf, roleName, roleGroup, containerId, - getClusterName()); - Map replaceTokens = providerUtils.filterSiteOptions( - appConf.getComponent(roleGroup).options, standardTokens); - replaceTokens.putAll(standardTokens); - - String roleNameKey = providerUtils.getNameKey(roleName, roleGroup, - appConf); - String roleNameIPKey = null; - if (roleNameKey != null) { - replaceTokens.put(String.format(HOST_KEY_FORMAT, roleNameKey), thisHost); - roleNameIPKey = Pattern.quote(String.format(IP_KEY_FORMAT, roleNameKey)); - } else { - // should not happen, but log if it does - log.info("Not replacing HOST or IP tokens because key was null for {}", - roleName); - } - String roleGroupKey = providerUtils.getGroupKey(roleGroup, appConf); - String roleGroupIPKey = null; - if (roleGroupKey != null) { - if (roleNameKey == null || !roleGroupKey.equals(roleNameKey)) { - replaceTokens.put(String.format(HOST_KEY_FORMAT, roleGroupKey), - thisHost); - roleGroupIPKey = Pattern.quote(String.format(IP_KEY_FORMAT, - roleGroupKey)); - } - } else { - // should not happen, but log if it does - log.info("Not replacing HOST or IP tokens because key was null for {}", - roleGroup); - } - replaceTokens.put("${THIS_HOST}", thisHost); - - for (Entry export : exports.entrySet()) { - String value = export.getValue(); - // replace host names and site properties - for (Map.Entry entry : replaceTokens.entrySet()) { - String token = entry.getKey(); - if (value.contains(token)) { - value = value.replaceAll(Pattern.quote(token), entry.getValue()); - } - } - Set values = new HashSet<>(); - for (String ip : ips) { - values.add(substituteIP(roleNameIPKey, roleGroupIPKey, ip, value)); - } - for (String exportValue : values) { - if (exportValue.contains(VARIABLE_INDICATOR)) { - // not all variables have been substituted, so do not export - continue; - } - ExportEntry entry = new ExportEntry(); - entry.setContainerId(containerId); - entry.setLevel(APPLICATION_TAG); - entry.setValue(exportValue); - entry.setUpdatedTime(new Date().toString()); - Set exportEntries = getExportEntries(export.getKey()); - exportEntries.add(entry); - log.info("Preparing to publish for {}. Key {} and Value {}", - roleName, export.getKey(), entry); - } - } - if (!exportMap.isEmpty()) { - providerUtils.publishExportGroup(exportMap, getAmState(), EXPORT_GROUP); - } - } - - protected String substituteIP(String roleNameIPKey, String roleGroupIPKey, - String ip, String value) { - if (roleNameIPKey != null) { - value = value.replaceAll(roleNameIPKey, ip); - } - if (roleGroupIPKey != null) { - value = value.replaceAll(roleGroupIPKey, ip); - } - return value; - } - - protected Set getExportEntries(String key) { - if (!this.exportMap.containsKey(key)) { - synchronized (this.exportMap) { - if (!this.exportMap.containsKey(key)) { - this.exportMap.put(key, Collections.newSetFromMap( - new ConcurrentHashMap<>())); - } - } - } - - return this.exportMap.get(key); - } - - @Override - public Map buildMonitorDetails(ClusterDescription clusterDesc) { - Map details = super.buildMonitorDetails(clusterDesc); - buildRoleHostDetails(details); - return details; - } - - private void buildRoleHostDetails(Map details) { - for (Map.Entry> entry : - getAmState().getRoleClusterNodeMapping().entrySet()) { - details.put(entry.getKey() + " Host(s)/Container(s)", - new MonitorDetail(providerUtils.getHostsList( - entry.getValue().values(), false).toString(), false)); - } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java deleted file mode 100644 index b58d3aaeaa3..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java +++ /dev/null @@ -1,305 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.slideram; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.slider.api.InternalKeys; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.api.RoleKeys; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.MapOperations; -import org.apache.slider.core.exceptions.BadClusterStateException; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.launch.AbstractLauncher; -import org.apache.slider.core.launch.JavaCommandLineBuilder; -import org.apache.slider.providers.AbstractClientProvider; -import org.apache.slider.providers.PlacementPolicy; -import org.apache.slider.providers.ProviderRole; -import org.apache.slider.providers.ProviderUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES; - -/** - * handles the setup of the Slider AM. - * This keeps aspects of role, cluster validation and Clusterspec setup - * out of the core slider client - */ -public class SliderAMClientProvider extends AbstractClientProvider - implements SliderKeys { - - - protected static final Logger log = - LoggerFactory.getLogger(SliderAMClientProvider.class); - protected static final String NAME = "SliderAM"; - public static final String INSTANCE_RESOURCE_BASE = PROVIDER_RESOURCE_BASE_ROOT + - "slideram/instance/"; - public static final String INTERNAL_JSON = - INSTANCE_RESOURCE_BASE + "internal.json"; - public static final String APPCONF_JSON = - INSTANCE_RESOURCE_BASE + "appconf.json"; - public static final String RESOURCES_JSON = - INSTANCE_RESOURCE_BASE + "resources.json"; - - public SliderAMClientProvider(Configuration conf) { - super(conf); - } - - /** - * List of roles - */ - public static final List ROLES = - new ArrayList(); - - public static final int KEY_AM = ROLE_AM_PRIORITY_INDEX; - - public static final ProviderRole APPMASTER = - new ProviderRole(COMPONENT_AM, KEY_AM, - PlacementPolicy.EXCLUDE_FROM_FLEXING, - ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD, - 0, ""); - - /** - * Initialize role list - */ - static { - ROLES.add(APPMASTER); - } - - @Override - public String getName() { - return NAME; - } - - @Override - public List getRoles() { - return ROLES; - } - - - @Override //Client - public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem, - String clustername, - Configuration configuration, - AggregateConf instanceDefinition, - Path clusterDirPath, - Path generatedConfDirPath, - boolean secure) - throws SliderException, IOException { - - super.preflightValidateClusterConfiguration(sliderFileSystem, clustername, configuration, instanceDefinition, clusterDirPath, generatedConfDirPath, secure); - //add a check for the directory being writeable by the current user - String - dataPath = instanceDefinition.getInternalOperations() - .getGlobalOptions() - .getMandatoryOption( - InternalKeys.INTERNAL_DATA_DIR_PATH); - - Path path = new Path(dataPath); - sliderFileSystem.verifyDirectoryWriteAccess(path); - Path historyPath = new Path(clusterDirPath, SliderKeys.HISTORY_DIR_NAME); - sliderFileSystem.verifyDirectoryWriteAccess(historyPath); - } - - /** - * Verify that an instance definition is considered valid by the provider - * @param instanceDefinition instance definition - * @throws SliderException if the configuration is not valid - */ - public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws - SliderException { - - super.validateInstanceDefinition(instanceDefinition, fs); - - // make sure there is no negative entry in the instance count - Map> instanceMap = - instanceDefinition.getResources().components; - for (Map.Entry> entry : instanceMap.entrySet()) { - MapOperations mapOperations = new MapOperations(entry); - int instances = mapOperations.getOptionInt(COMPONENT_INSTANCES, 0); - if (instances < 0) { - throw new BadClusterStateException( - "Component %s has negative instance count: %d", - mapOperations.name, - instances); - } - } - } - - /** - * The Slider AM sets up all the dependency JARs above slider.jar itself - * {@inheritDoc} - */ - public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem, - Configuration serviceConf, - AbstractLauncher launcher, - AggregateConf instanceDescription, - Path snapshotConfDirPath, - Path generatedConfDirPath, - Configuration clientConfExtras, - String libdir, - Path tempPath, boolean miniClusterTestRun) - throws IOException, SliderException { - - Map providerResources = new HashMap<>(); - - ProviderUtils.addProviderJar(providerResources, - this, - SLIDER_JAR, - fileSystem, - tempPath, - libdir, - miniClusterTestRun); - - log.info("Loading all dependencies for AM."); - // If slider.tar.gz is available in hdfs use it, else upload all jars - Path dependencyLibTarGzip = fileSystem.getDependencyTarGzip(); - if (fileSystem.isFile(dependencyLibTarGzip)) { - SliderUtils.putAmTarGzipAndUpdate(providerResources, fileSystem); - } else { - for (String libDirProp : SliderUtils.getLibDirs()) { - ProviderUtils.addAllDependencyJars(providerResources, - fileSystem, - tempPath, - libdir, - libDirProp); - - } - } - addKeytabResourceIfNecessary(fileSystem, - instanceDescription, - providerResources); - - launcher.addLocalResources(providerResources); - - //also pick up all env variables from a map - launcher.copyEnvVars( - instanceDescription.getInternalOperations().getOrAddComponent( - SliderKeys.COMPONENT_AM)); - } - - /** - * If the cluster is secure, and an HDFS installed keytab is available for AM - * authentication, add this keytab as a local resource for the AM launch. - * - * @param fileSystem - * @param instanceDescription - * @param providerResources - * @throws IOException - * @throws BadConfigException if there's no keytab and it is explicitly required. - */ - protected void addKeytabResourceIfNecessary(SliderFileSystem fileSystem, - AggregateConf instanceDescription, - Map providerResources) - throws IOException, BadConfigException { - if (UserGroupInformation.isSecurityEnabled()) { - String keytabPathOnHost = instanceDescription.getAppConfOperations() - .getComponent(SliderKeys.COMPONENT_AM).get( - SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); - if (SliderUtils.isUnset(keytabPathOnHost)) { - String amKeytabName = instanceDescription.getAppConfOperations() - .getComponent(SliderKeys.COMPONENT_AM).get( - SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME); - String keytabDir = instanceDescription.getAppConfOperations() - .getComponent(SliderKeys.COMPONENT_AM).get( - SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR); - Path keytabPath = fileSystem.buildKeytabPath(keytabDir, amKeytabName, - instanceDescription.getName()); - if (fileSystem.getFileSystem().exists(keytabPath)) { - LocalResource keytabRes = fileSystem.createAmResource(keytabPath, - LocalResourceType.FILE); - - providerResources.put(SliderKeys.KEYTAB_DIR + "/" + - amKeytabName, keytabRes); - } else { - log.warn("No keytab file was found at {}.", keytabPath); - if (getConf().getBoolean(KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) { - throw new BadConfigException("No keytab file was found at %s.", keytabPath); - - } else { - log.warn("The AM will be " - + "started without a kerberos authenticated identity. " - + "The application is therefore not guaranteed to remain " - + "operational beyond 24 hours."); - } - } - } - } - } - - /** - * Update the AM resource with any local needs - * @param capability capability to update - */ - public void prepareAMResourceRequirements(MapOperations sliderAM, - Resource capability) { - capability.setMemory(sliderAM.getOptionInt( - ResourceKeys.YARN_MEMORY, - capability.getMemory())); - capability.setVirtualCores( - sliderAM.getOptionInt(ResourceKeys.YARN_CORES, capability.getVirtualCores())); - } - - /** - * Extract any JVM options from the cluster specification and - * add them to the command line - */ - public void addJVMOptions(AggregateConf aggregateConf, - JavaCommandLineBuilder cmdLine) - throws BadConfigException { - - MapOperations sliderAM = - aggregateConf.getAppConfOperations().getMandatoryComponent( - SliderKeys.COMPONENT_AM); - cmdLine.forceIPv4().headless(); - String heap = sliderAM.getOption(RoleKeys.JVM_HEAP, - DEFAULT_JVM_HEAP); - cmdLine.setJVMHeap(heap); - String jvmopts = sliderAM.getOption(RoleKeys.JVM_OPTS, ""); - if (SliderUtils.isSet(jvmopts)) { - cmdLine.add(jvmopts); - } - } - - - @Override - public void prepareInstanceConfiguration(AggregateConf aggregateConf) - throws SliderException, IOException { - mergeTemplates(aggregateConf, - INTERNAL_JSON, RESOURCES_JSON, APPCONF_JSON - ); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java deleted file mode 100644 index c021b809377..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.slideram; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.tools.ConfigHelper; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.MapOperations; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.launch.ContainerLauncher; -import org.apache.slider.core.registry.docstore.PublishedConfiguration; -import org.apache.slider.core.registry.info.CustomRegistryConstants; -import org.apache.slider.providers.AbstractProviderService; -import org.apache.slider.providers.ProviderCore; -import org.apache.slider.providers.ProviderRole; -import org.apache.slider.providers.agent.AgentKeys; -import org.apache.slider.server.appmaster.PublishedArtifacts; -import org.apache.slider.server.appmaster.web.rest.RestPaths; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; - -import static org.apache.slider.server.appmaster.web.rest.RestPaths.*; - -/** - * Exists just to move some functionality out of AppMaster into a peer class - * of the actual service provider doing the real work - */ -public class SliderAMProviderService extends AbstractProviderService implements - ProviderCore, - AgentKeys, - SliderKeys { - - public SliderAMProviderService() { - super("SliderAMProviderService"); - } - - @Override - public String getHumanName() { - return "Slider Application"; - } - - @Override - public Configuration loadProviderConfigurationInformation(File confDir) throws - BadCommandArgumentsException, - IOException { - return null; - } - - @Override - public void buildContainerLaunchContext(ContainerLauncher containerLauncher, - AggregateConf instanceDefinition, - Container container, - ProviderRole role, - SliderFileSystem sliderFileSystem, - Path generatedConfPath, - MapOperations resourceComponent, - MapOperations appComponent, - Path containerTmpDirPath) throws IOException, SliderException { - } - - @Override - public List getRoles() { - return new ArrayList<>(0); - } - - @Override - public void validateInstanceDefinition(AggregateConf instanceDefinition) throws - SliderException { - - } - - @Override - public void applyInitialRegistryDefinitions(URL amWebURI, - ServiceRecord serviceRecord) - throws IOException { - super.applyInitialRegistryDefinitions(amWebURI, - serviceRecord); - // now publish site.xml files - YarnConfiguration defaultYarnConfig = new YarnConfiguration(); - amState.getPublishedSliderConfigurations().put( - PublishedArtifacts.COMPLETE_CONFIG, - new PublishedConfiguration( - "Complete slider application settings", - getConfig(), getConfig())); - amState.getPublishedSliderConfigurations().put( - PublishedArtifacts.YARN_SITE_CONFIG, - new PublishedConfiguration( - "YARN site settings", - ConfigHelper.loadFromResource("yarn-site.xml"), - defaultYarnConfig) ); - - amState.getPublishedSliderConfigurations().put( - PublishedArtifacts.CORE_SITE_CONFIG, - new PublishedConfiguration( - "Core site settings", - ConfigHelper.loadFromResource("core-site.xml"), - defaultYarnConfig) ); - amState.getPublishedSliderConfigurations().put( - PublishedArtifacts.HDFS_SITE_CONFIG, - new PublishedConfiguration( - "HDFS site settings", - ConfigHelper.loadFromResource("hdfs-site.xml"), - new HdfsConfiguration(true)) ); - - - try { - - URL managementAPI = new URL(amWebURI, RELATIVE_PATH_MANAGEMENT); - URL registryREST = new URL(amWebURI, RELATIVE_PATH_REGISTRY); - - URL publisherURL = new URL(amWebURI, RELATIVE_PATH_PUBLISHER); - - // Set the configurations URL. - - String configurationsURL = SliderUtils.appendToURL( - publisherURL.toExternalForm(), RestPaths.SLIDER_CONFIGSET); - String exportsURL = SliderUtils.appendToURL( - publisherURL.toExternalForm(), RestPaths.SLIDER_EXPORTS); - - serviceRecord.addExternalEndpoint( - RegistryTypeUtils.webEndpoint( - CustomRegistryConstants.WEB_UI, amWebURI.toURI())); - - serviceRecord.addExternalEndpoint( - RegistryTypeUtils.webEndpoint( - CustomRegistryConstants.AM_REST_BASE, amWebURI.toURI())); - - serviceRecord.addExternalEndpoint( - RegistryTypeUtils.restEndpoint( - CustomRegistryConstants.MANAGEMENT_REST_API, - managementAPI.toURI())); - serviceRecord.addExternalEndpoint( - RegistryTypeUtils.restEndpoint( - CustomRegistryConstants.PUBLISHER_REST_API, - publisherURL.toURI())); - serviceRecord.addExternalEndpoint( - RegistryTypeUtils.restEndpoint( - CustomRegistryConstants.REGISTRY_REST_API, - registryREST.toURI())); - serviceRecord.addExternalEndpoint( - RegistryTypeUtils.restEndpoint( - CustomRegistryConstants.PUBLISHER_CONFIGURATIONS_API, - new URI(configurationsURL))); - serviceRecord.addExternalEndpoint( - RegistryTypeUtils.restEndpoint( - CustomRegistryConstants.PUBLISHER_EXPORTS_API, - new URI(exportsURL))); - - } catch (URISyntaxException e) { - throw new IOException(e); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java index 3cfe1674359..70eab715c7b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java @@ -18,16 +18,12 @@ package org.apache.slider.server.appmaster; -import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.slider.common.SliderKeys; +import org.apache.slider.api.resource.Application; import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.launch.ContainerLauncher; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderService; @@ -37,8 +33,8 @@ import org.apache.slider.server.appmaster.actions.QueueAccess; import org.apache.slider.server.appmaster.state.ContainerAssignment; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.state.RoleStatus; -import org.apache.slider.server.services.workflow.WorkflowExecutorService; import org.apache.slider.server.services.workflow.ServiceThreadFactory; +import org.apache.slider.server.services.workflow.WorkflowExecutorService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,16 +69,6 @@ public class RoleLaunchService */ private final SliderFileSystem fs; - /** - * Path in the launch filesystem that refers to a configuration directory - * -the interpretation of it is left to the Provider - */ - private final Path generatedConfDirPath; - /** - * Path in the launch filesystem that refers to a temp directory - * which will be cleaned up at (some) time in the future - */ - private final Path launcherTmpDirPath; private Map envVars; @@ -91,21 +77,13 @@ public class RoleLaunchService * @param queueAccess * @param provider the provider * @param fs filesystem - * @param generatedConfDirPath path in the FS for the generated dir * @param envVars environment variables - * @param launcherTmpDirPath path for a temporary data in the launch process */ - public RoleLaunchService(QueueAccess queueAccess, - ProviderService provider, - SliderFileSystem fs, - Path generatedConfDirPath, - Map envVars, - Path launcherTmpDirPath) { + public RoleLaunchService(QueueAccess queueAccess, ProviderService provider, + SliderFileSystem fs, Map envVars) { super(ROLE_LAUNCH_SERVICE); this.actionQueue = queueAccess; this.fs = fs; - this.generatedConfDirPath = generatedConfDirPath; - this.launcherTmpDirPath = launcherTmpDirPath; this.provider = provider; this.envVars = envVars; } @@ -120,23 +98,13 @@ public class RoleLaunchService /** * Start an asychronous launch operation * @param assignment container assignment - * @param clusterSpec cluster spec to use for template * @param credentials credentials to use */ public void launchRole(ContainerAssignment assignment, - AggregateConf clusterSpec, - Credentials credentials) { - RoleStatus role = assignment.role; - String roleName = role.getName(); - String roleGroup = role.getGroup(); - // prelaunch safety check - Preconditions.checkArgument(provider.isSupportedRole(roleName)); + Application application, Credentials credentials) { RoleLaunchService.RoleLauncher launcher = - new RoleLaunchService.RoleLauncher(assignment, - clusterSpec, - clusterSpec.getResourceOperations().getOrAddComponent(roleGroup), - clusterSpec.getAppConfOperations().getOrAddComponent(roleGroup), - credentials); + new RoleLaunchService.RoleLauncher(assignment, application, + credentials); execute(launcher); } @@ -148,35 +116,21 @@ public class RoleLaunchService private final ContainerAssignment assignment; // Allocated container public final Container container; - private final MapOperations resourceComponent; - private final MapOperations appComponent; - private final AggregateConf instanceDefinition; + public final Application application; public final ProviderRole role; private final Credentials credentials; - private Exception raisedException; public RoleLauncher(ContainerAssignment assignment, - AggregateConf instanceDefinition, - MapOperations resourceComponent, - MapOperations appComponent, + Application application, Credentials credentials) { this.assignment = assignment; this.credentials = credentials; this.container = assignment.container; RoleStatus roleStatus = assignment.role; - - assert resourceComponent != null; - assert appComponent != null; ProviderRole providerRole = roleStatus.getProviderRole(); - assert providerRole != null; this.role = providerRole; - this.resourceComponent = resourceComponent; - this.appComponent = appComponent; - this.instanceDefinition = instanceDefinition; - } + this.application = application; - public Exception getRaisedException() { - return raisedException; } @Override @@ -196,62 +150,46 @@ public class RoleLaunchService containerLauncher.setupUGI(); containerLauncher.putEnv(envVars); - log.debug("Launching container {} into role {}", - container.getId(), - role.name); + log.info("Launching container {} into RoleName = {}, RoleGroup = {}", + container.getId(), role.name, role.group); - //now build up the configuration data - Path containerTmpDirPath = - new Path(launcherTmpDirPath, container.getId().toString()); - provider.buildContainerLaunchContext(containerLauncher, - instanceDefinition, - container, - role, - fs, - generatedConfDirPath, - resourceComponent, - appComponent, - containerTmpDirPath); + provider.buildContainerLaunchContext(containerLauncher, application, + container, role, fs); RoleInstance instance = new RoleInstance(container); String[] envDescription = containerLauncher.dumpEnvToString(); String commandsAsString = containerLauncher.getCommandsAsString(); - log.info("Starting container with command: {}", - commandsAsString); + log.info("Starting container with command: {}", commandsAsString); + instance.providerRole = role; instance.command = commandsAsString; instance.role = role.name; instance.group = role.group; instance.roleId = role.id; - instance.appVersion = instanceDefinition.getAppConfOperations() - .getGlobalOptions().get(SliderKeys.APP_VERSION); instance.environment = envDescription; - int delay = appComponent.getOptionInt( - AgentKeys.KEY_CONTAINER_LAUNCH_DELAY, 0); - int maxDelay = - getConfig().getInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, - YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS); + long delay = role.component.getConfiguration() + .getPropertyLong(AgentKeys.KEY_CONTAINER_LAUNCH_DELAY, 0); + long maxDelay = getConfig() + .getLong(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, + YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS); if (delay > maxDelay/1000) { log.warn("Container launch delay of {} exceeds the maximum allowed of" + " {} seconds. Delay will not be utilized.", delay, maxDelay/1000); delay = 0; } - log.info("Container launch delay for {} set to {} seconds", - role.name, delay); - actionQueue.schedule(new ActionStartContainer("starting " + role.name, - container, - containerLauncher.completeContainerLaunch(), - instance, - delay, - TimeUnit.SECONDS)); + log.info("Container launch delay for {} set to {} seconds", role.name, + delay); + actionQueue.schedule( + new ActionStartContainer("starting " + role.name, container, + containerLauncher.completeContainerLaunch(), instance, delay, + TimeUnit.SECONDS)); } catch (Exception e) { - log.error("Exception thrown while trying to start {}: {}", - role.name, e, e); - raisedException = e; + log.error("Exception thrown while trying to start " + role.name + + " container = " + container.getId() + " on host " + container + .getNodeId(), e); } } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index f67ea58a9e3..4922c2dcf84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -30,6 +30,8 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.io.Text; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; @@ -62,6 +64,7 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; import org.apache.hadoop.yarn.client.api.async.NMClientAsync; import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; @@ -77,13 +80,12 @@ import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.WebAppException; import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; -import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.InternalKeys; import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.RoleKeys; -import org.apache.slider.api.StatusKeys; +import org.apache.slider.api.proto.Messages; import org.apache.slider.api.proto.SliderClusterAPI; -import org.apache.slider.client.SliderYarnClientImpl; +import org.apache.slider.api.resource.Application; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.params.AbstractActionArgs; @@ -95,10 +97,7 @@ import org.apache.slider.common.tools.PortScanner; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.common.tools.SliderVersionInfo; -import org.apache.slider.core.buildutils.InstanceIO; import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTree; -import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.core.exceptions.SliderException; @@ -109,13 +108,12 @@ import org.apache.slider.core.main.ExitCodeProvider; import org.apache.slider.core.main.LauncherExitCodes; import org.apache.slider.core.main.RunService; import org.apache.slider.core.main.ServiceLauncher; +import org.apache.slider.core.persist.JsonSerDeser; import org.apache.slider.core.registry.info.CustomRegistryConstants; import org.apache.slider.providers.ProviderCompleted; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderService; import org.apache.slider.providers.SliderProviderFactory; -import org.apache.slider.providers.slideram.SliderAMClientProvider; -import org.apache.slider.providers.slideram.SliderAMProviderService; import org.apache.slider.server.appmaster.actions.ActionHalt; import org.apache.slider.server.appmaster.actions.ActionRegisterServiceInstance; import org.apache.slider.server.appmaster.actions.ActionStopSlider; @@ -136,7 +134,6 @@ import org.apache.slider.server.appmaster.monkey.ChaosKillContainer; import org.apache.slider.server.appmaster.monkey.ChaosMonkeyService; import org.apache.slider.server.appmaster.operations.AbstractRMOperation; import org.apache.slider.server.appmaster.operations.AsyncRMOperationHandler; -import org.apache.slider.server.appmaster.operations.ProviderNotifyingOperationHandler; import org.apache.slider.server.appmaster.operations.RMOperationHandler; import org.apache.slider.server.appmaster.rpc.RpcBinder; import org.apache.slider.server.appmaster.rpc.SliderAMPolicyProvider; @@ -146,6 +143,7 @@ import org.apache.slider.server.appmaster.security.SecurityConfiguration; import org.apache.slider.server.appmaster.state.AppState; import org.apache.slider.server.appmaster.state.AppStateBindingInfo; import org.apache.slider.server.appmaster.state.ContainerAssignment; +import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector; import org.apache.slider.server.appmaster.state.ProviderAppState; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.web.SliderAMWebApp; @@ -161,18 +159,20 @@ import org.apache.slider.server.services.workflow.ServiceThreadFactory; import org.apache.slider.server.services.workflow.WorkflowExecutorService; import org.apache.slider.server.services.workflow.WorkflowRpcService; import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; +import org.codehaus.jackson.map.PropertyNamingStrategy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; -import java.net.URI; import java.net.URL; import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -242,8 +242,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized") private RMOperationHandler rmOperationHandler; - - private RMOperationHandler providerRMOperationHandler; /** Handle to communicate with the Node Manager*/ @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized") @@ -252,7 +250,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService /** * Credentials for propagating down to launched containers */ - private Credentials containerCredentials; + private Credentials containerCredentials = new Credentials(); /** * Slider IPC: Real service handler @@ -320,13 +318,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService */ private final AtomicBoolean initCompleted = new AtomicBoolean(false); - /** - * Flag to set if the process exit code was set before shutdown started - */ - @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized") - private boolean spawnedProcessExitedBeforeShutdownTriggered; - - /** Arguments passed in : raw*/ @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized") private SliderAMArgs serviceArgs; @@ -371,7 +362,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService private SliderAMWebApp webApp; @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized") private InetSocketAddress rpcServiceAddress; - private SliderAMProviderService sliderAMProvider; /** * Executor. @@ -398,12 +388,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService */ private boolean securityEnabled; private ContentCache contentCache; + private static final JsonSerDeser jsonSerDeser = + new JsonSerDeser(Application.class, + PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES); /** * resource limits */ private Resource maximumResourceCapability; - + private Application application; /** * Service Constructor */ @@ -586,84 +579,31 @@ public class SliderAppMaster extends AbstractSliderLaunchedService /** * Create and run the cluster. - * @param clustername cluster name + * @param appName cluster name * @return exit code * @throws Throwable on a failure */ - private int createAndRunCluster(String clustername) throws Throwable { - - //load the cluster description from the cd argument - String sliderClusterDir = serviceArgs.getSliderClusterURI(); - URI sliderClusterURI = new URI(sliderClusterDir); - Path clusterDirPath = new Path(sliderClusterURI); - log.info("Application defined at {}", sliderClusterURI); + private int createAndRunCluster(String appName) throws Throwable { + Path appDir = new Path((serviceArgs.getAppDefDir())); SliderFileSystem fs = getClusterFS(); - - // build up information about the running application -this - // will be passed down to the cluster status - MapOperations appInformation = new MapOperations(); - - AggregateConf instanceDefinition = - InstanceIO.loadInstanceDefinitionUnresolved(fs, clusterDirPath); - instanceDefinition.setName(clustername); - - log.info("Deploying cluster {}:", instanceDefinition); - - // and resolve it - AggregateConf resolvedInstance = new AggregateConf( instanceDefinition); - resolvedInstance.resolve(); - - stateForProviders.setApplicationName(clustername); - + fs.setAppDir(appDir); + Path appJson = new Path(appDir, appName + ".json"); + log.info("Loading application definition from " + appJson); + application = jsonSerDeser.load(fs.getFileSystem(), appJson); + log.info("Application Json: " + application); + stateForProviders.setApplicationName(appName); Configuration serviceConf = getConfig(); - // extend AM configuration with component resource - MapOperations amConfiguration = resolvedInstance - .getAppConfOperations().getComponent(COMPONENT_AM); - // and patch configuration with prefix - if (amConfiguration != null) { - Map sliderAppConfKeys = amConfiguration.prefixedWith("slider."); - for (Map.Entry entry : sliderAppConfKeys.entrySet()) { - String k = entry.getKey(); - String v = entry.getValue(); - boolean exists = serviceConf.get(k) != null; - log.info("{} {} to {}", (exists ? "Overwriting" : "Setting"), k, v); - serviceConf.set(k, v); - } - } - - securityConfiguration = new SecurityConfiguration(serviceConf, resolvedInstance, clustername); // obtain security state - securityEnabled = securityConfiguration.isSecurityEnabled(); // set the global security flag for the instance definition - instanceDefinition.getAppConfOperations().set(KEY_SECURITY_ENABLED, securityEnabled); - // triggers resolution and snapshotting for agent - appState.setInitialInstanceDefinition(instanceDefinition); - - File confDir = getLocalConfDir(); - if (!confDir.exists() || !confDir.isDirectory()) { - log.info("Conf dir {} does not exist.", confDir); - File parentFile = confDir.getParentFile(); - log.info("Parent dir {}:\n{}", parentFile, SliderUtils.listDir(parentFile)); - } - //get our provider - MapOperations globalInternalOptions = getGlobalInternalOptions(); - String providerType = globalInternalOptions.getMandatoryOption( - InternalKeys.INTERNAL_PROVIDER_NAME); - log.info("Cluster provider type is {}", providerType); SliderProviderFactory factory = - SliderProviderFactory.createSliderProviderFactory(providerType); + SliderProviderFactory.createSliderProviderFactory("docker"); providerService = factory.createServerProvider(); // init the provider BUT DO NOT START IT YET initAndAddService(providerService); - providerRMOperationHandler = new ProviderNotifyingOperationHandler(providerService); - - // create a slider AM provider - sliderAMProvider = new SliderAMProviderService(); - initAndAddService(sliderAMProvider); - + InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf); log.info("RM is at {}", rmSchedulerAddress); yarnRPC = YarnRPC.create(serviceConf); @@ -689,10 +629,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService ApplicationId appid = appAttemptID.getApplicationId(); log.info("AM for ID {}", appid.getId()); - appInformation.put(StatusKeys.INFO_AM_CONTAINER_ID, appMasterContainerID.toString()); - appInformation.put(StatusKeys.INFO_AM_APP_ID, appid.toString()); - appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID, appAttemptID.toString()); - Map envVars; List liveContainers; @@ -731,28 +667,22 @@ public class SliderAppMaster extends AbstractSliderLaunchedService } //bring up the Slider RPC service - buildPortScanner(instanceDefinition); - startSliderRPCServer(instanceDefinition); + buildPortScanner(); + startSliderRPCServer(); rpcServiceAddress = rpcService.getConnectAddress(); appMasterHostname = rpcServiceAddress.getAddress().getCanonicalHostName(); appMasterRpcPort = rpcServiceAddress.getPort(); appMasterTrackingUrl = null; log.info("AM Server is listening at {}:{}", appMasterHostname, appMasterRpcPort); - appInformation.put(StatusKeys.INFO_AM_HOSTNAME, appMasterHostname); - appInformation.set(StatusKeys.INFO_AM_RPC_PORT, appMasterRpcPort); log.info("Starting Yarn registry"); registryOperations = startRegistryOperationsService(); log.info(registryOperations.toString()); //build the role map - List providerRoles = new ArrayList<>(providerService.getRoles()); - providerRoles.addAll(SliderAMClientProvider.ROLES); - + List providerRoles = Collections.EMPTY_LIST; // Start up the WebApp and track the URL for it - MapOperations component = instanceDefinition.getAppConfOperations() - .getComponent(SliderKeys.COMPONENT_AM); // Web service endpoints: initialize WebAppApiImpl webAppApi = @@ -760,9 +690,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService stateForProviders, providerService, registryOperations, metricsAndMonitoring, - actionQueues, - this, - contentCache); + actionQueues); initAMFilterOptions(serviceConf); int webAppPort = deployWebApplication(webAppApi); @@ -770,9 +698,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService String scheme = WebAppUtils.HTTP_PREFIX; appMasterTrackingUrl = scheme + appMasterHostname + ":" + webAppPort; - appInformation.put(StatusKeys.INFO_AM_WEB_URL, appMasterTrackingUrl + "/"); - appInformation.set(StatusKeys.INFO_AM_WEB_PORT, webAppPort); - // ***************************************************** // Register self with ResourceManager // This will start heartbeating to the RM @@ -785,6 +710,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService appMasterTrackingUrl); maximumResourceCapability = amRegistrationData.getMaximumResourceCapability(); + //TODO should not read local configs !!! int minMemory = serviceConf.getInt(RM_SCHEDULER_MINIMUM_ALLOCATION_MB, DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); // validate scheduler vcores allocation setting @@ -798,11 +724,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // the max value as part of its lookup rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient, maximumResourceCapability); - // set the RM-defined maximum cluster values - appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(maxCores)); - appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(maxMemory)); - - processAMCredentials(securityConfiguration); +// processAMCredentials(securityConfiguration); if (securityEnabled) { secretManager.setMasterKey( @@ -817,7 +739,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // principal. Can do so now since AM registration with RM above required // tokens associated to principal String principal = securityConfiguration.getPrincipal(); - File localKeytabFile = securityConfiguration.getKeytabFile(instanceDefinition); + //TODO read key tab file from slider-am.xml + File localKeytabFile = + securityConfiguration.getKeytabFile(new AggregateConf()); // Now log in... login(principal, localKeytabFile); // obtain new FS reference that should be kerberos based and different @@ -829,10 +753,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // YARN client. // Important: this is only valid at startup, and must be executed within // the right UGI context. Use with care. - SliderYarnClientImpl yarnClient = null; + YarnClient yarnClient = null; List nodeReports; try { - yarnClient = new SliderYarnClientImpl(); + yarnClient = YarnClient.createYarnClient(); yarnClient.init(getConfig()); yarnClient.start(); nodeReports = getNodeReports(yarnClient); @@ -856,45 +780,23 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // extract container list liveContainers = amRegistrationData.getContainersFromPreviousAttempts(); - - //now validate the installation - Configuration providerConf = - providerService.loadProviderConfigurationInformation(confDir); - - providerService.initializeApplicationConfiguration(instanceDefinition, - fs, null); - - providerService.validateApplicationConfiguration(instanceDefinition, - confDir, - securityEnabled); + DefaultMetricsSystem.initialize("SliderAppMaster"); //determine the location for the role history data - Path historyDir = new Path(clusterDirPath, HISTORY_DIR_NAME); + Path historyDir = new Path(appDir, HISTORY_DIR_NAME); //build the instance AppStateBindingInfo binding = new AppStateBindingInfo(); - binding.instanceDefinition = instanceDefinition; binding.serviceConfig = serviceConf; - binding.publishedProviderConf = providerConf; binding.roles = providerRoles; binding.fs = fs.getFileSystem(); binding.historyPath = historyDir; binding.liveContainers = liveContainers; - binding.applicationInfo = appInformation; - binding.releaseSelector = providerService.createContainerReleaseSelector(); + binding.releaseSelector = new MostRecentContainerReleaseSelector(); binding.nodeReports = nodeReports; + binding.application = application; appState.buildInstance(binding); - providerService.rebuildContainerDetails(liveContainers, - instanceDefinition.getName(), appState.getRolePriorityMap()); - - // add the AM to the list of nodes in the cluster - - appState.buildAppMasterNode(appMasterContainerID, - appMasterHostname, - webAppPort, - appMasterHostname + ":" + webAppPort); - // build up environment variables that the AM wants set in every container // irrespective of provider and role. envVars = new HashMap<>(); @@ -908,8 +810,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService } String rolesTmpSubdir = appMasterContainerID.toString() + "/roles"; - String amTmpDir = globalInternalOptions.getMandatoryOption(InternalKeys.INTERNAL_AM_TMP_DIR); - + String amTmpDir = "/tmp"; + //TODO read tmpDir from slider-am.xml Path tmpDirPath = new Path(amTmpDir); Path launcherTmpDirPath = new Path(tmpDirPath, rolesTmpSubdir); fs.getFileSystem().mkdirs(launcherTmpDirPath); @@ -917,29 +819,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService //launcher service launchService = new RoleLaunchService(actionQueues, providerService, - fs, - new Path(getGeneratedConfDir()), - envVars, - launcherTmpDirPath); + fs, envVars); deployChildService(launchService); - appState.noteAMLaunched(); - - //Give the provider access to the state, and AM - providerService.bind(stateForProviders, actionQueues, liveContainers); - sliderAMProvider.bind(stateForProviders, actionQueues, liveContainers); + providerService.setAMState(stateForProviders); // chaos monkey - maybeStartMonkey(); - - // setup token renewal and expiry handling for long lived apps -// if (!securityConfiguration.isKeytabProvided() && -// SliderUtils.isHadoopClusterSecure(getConfig())) { -// fsDelegationTokenManager = new FsDelegationTokenManager(actionQueues); -// fsDelegationTokenManager.acquireDelegationToken(getConfig()); -// } +// maybeStartMonkey(); // if not a secure cluster, extract the username -it will be // propagated to workers @@ -955,25 +843,21 @@ public class SliderAppMaster extends AbstractSliderLaunchedService log.info("Application Master Initialization Completed"); initCompleted.set(true); - scheduleFailureWindowResets(instanceDefinition.getResources()); - scheduleEscalation(instanceDefinition.getInternal()); + scheduleFailureWindowResets(application.getConfiguration()); + scheduleEscalation(application.getConfiguration()); try { // schedule YARN Registry registration - queue(new ActionRegisterServiceInstance(clustername, appid)); + queue(new ActionRegisterServiceInstance(appName, appid, application)); // log the YARN and web UIs log.info("RM Webapp address {}", serviceConf.get(YarnConfiguration.RM_WEBAPP_ADDRESS)); log.info("Slider webapp address {} proxied at {}", appMasterTrackingUrl, appMasterProxiedUrl); - - // Start the Slider AM provider - sliderAMProvider.start(); - // launch the real provider; this is expected to trigger a callback that // starts the node review process - launchProviderService(instanceDefinition, confDir); + launchProviderService(); // start handling any scheduled events @@ -1000,7 +884,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService * @throws InterruptedException */ private ApplicationAttemptReport getApplicationAttemptReport( - final SliderYarnClientImpl yarnClient) + final YarnClient yarnClient) throws YarnException, IOException, InterruptedException { Preconditions.checkNotNull(yarnClient, "Null Yarn client"); ApplicationAttemptReport report; @@ -1019,14 +903,14 @@ public class SliderAppMaster extends AbstractSliderLaunchedService } /** - * List the node reports: uses {@link SliderYarnClientImpl} as the login user + * List the node reports: uses {@link YarnClient} as the login user * @param yarnClient client to the RM * @return the node reports * @throws IOException * @throws YarnException * @throws InterruptedException */ - private List getNodeReports(final SliderYarnClientImpl yarnClient) + private List getNodeReports(final YarnClient yarnClient) throws IOException, YarnException, InterruptedException { Preconditions.checkNotNull(yarnClient, "Null Yarn client"); List nodeReports; @@ -1051,7 +935,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService * Creates and starts the web application, and adds a * WebAppService service under the AM, to ensure * a managed web application shutdown. - * @param webAppApi web app API instance + * @param webAppApi web application API instance * @return port the web application is deployed on * @throws IOException general problems starting the webapp (network, etc) * @throws WebAppException other issues @@ -1117,12 +1001,14 @@ public class SliderAppMaster extends AbstractSliderLaunchedService /** * Build up the port scanner. This may include setting a port range. */ - private void buildPortScanner(AggregateConf instanceDefinition) + private void buildPortScanner() throws BadConfigException { portScanner = new PortScanner(); - String portRange = instanceDefinition. - getAppConfOperations().getGlobalOptions(). - getOption(SliderKeys.KEY_ALLOWED_PORT_RANGE, "0"); + String portRange = "0"; + //TODO read from slider-am.xml +// String portRange = instanceDefinition. +// getAppConfOperations().getGlobalOptions(). +// getOption(SliderKeys.KEY_ALLOWED_PORT_RANGE, "0"); if (!"0".equals(portRange)) { portScanner.setPortRange(portRange); } @@ -1203,11 +1089,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService * @throws IOException */ public void registerServiceInstance(String instanceName, - ApplicationId appId) throws IOException { - - - // the registry is running, so register services - URL amWebURI = new URL(appMasterProxiedUrl); + ApplicationId appId, Application application) throws IOException { //Give the provider restricted access to the state, registry setupInitialRegistryPaths(); @@ -1218,7 +1100,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService instanceName, appAttemptID); providerService.bindToYarnRegistry(yarnRegistryOperations); - sliderAMProvider.bindToYarnRegistry(yarnRegistryOperations); // Yarn registry ServiceRecord serviceRecord = new ServiceRecord(); @@ -1231,19 +1112,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService RegistryTypeUtils.ipcEndpoint( CustomRegistryConstants.AM_IPC_PROTOCOL, rpcServiceAddress)); - - // internal services - sliderAMProvider.applyInitialRegistryDefinitions(amWebURI, - serviceRecord); - - // provider service dynamic definitions. - providerService.applyInitialRegistryDefinitions(amWebURI, serviceRecord); - // set any provided attributes - setProvidedServiceRecordAttributes( - getInstanceDefinition().getAppConfOperations().getComponent( - SliderKeys.COMPONENT_AM), serviceRecord); + setUserProvidedServiceRecordAttributes(application.getConfiguration(), + serviceRecord); // register the service's entry log.info("Service Record \n{}", serviceRecord); @@ -1276,7 +1148,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService /** * Handler for {@link RegisterComponentInstance action} - * Register/re-register an ephemeral container that is already in the app state + * Register/re-register an ephemeral container that is already in the application state * @param id the component * @param description component description * @param type component type @@ -1291,32 +1163,36 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // this is where component registrations go log.info("Registering component {}", id); String cid = RegistryPathUtils.encodeYarnID(id.toString()); - ServiceRecord container = new ServiceRecord(); - container.set(YarnRegistryAttributes.YARN_ID, cid); - container.description = description; - container.set(YarnRegistryAttributes.YARN_PERSISTENCE, + ServiceRecord record = new ServiceRecord(); + record.set(YarnRegistryAttributes.YARN_ID, cid); + record.description = description; + record.set(YarnRegistryAttributes.YARN_PERSISTENCE, PersistencePolicies.CONTAINER); - MapOperations compOps = getInstanceDefinition().getAppConfOperations(). - getComponent(type); - setProvidedServiceRecordAttributes(compOps, container); + setUserProvidedServiceRecordAttributes( + instance.providerRole.component.getConfiguration(), record); try { - yarnRegistryOperations.putComponent(cid, container); + yarnRegistryOperations.putComponent(cid, record); } catch (IOException e) { log.warn("Failed to register container {}/{}: {}", id, description, e, e); return false; } + org.apache.slider.api.resource.Container container = + new org.apache.slider.api.resource.Container(); + container.setId(id.toString()); + container.setLaunchTime(new Date()); + container.setState(org.apache.slider.api.resource.ContainerState.INIT); + container.setBareHost(instance.host); + instance.providerRole.component.addContainer(container); return true; } - protected void setProvidedServiceRecordAttributes(MapOperations ops, - ServiceRecord record) { + protected void setUserProvidedServiceRecordAttributes( + org.apache.slider.api.resource.Configuration conf, ServiceRecord record) { String prefix = RoleKeys.SERVICE_RECORD_ATTRIBUTE_PREFIX; - for (Map.Entry entry : ops.entrySet()) { - if (entry.getKey().startsWith( - prefix)) { - String key = entry.getKey().substring( - prefix.length() + 1); + for (Map.Entry entry : conf.getProperties().entrySet()) { + if (entry.getKey().startsWith(prefix)) { + String key = entry.getKey().substring(prefix.length() + 1); record.set(key, entry.getValue().trim()); } } @@ -1365,35 +1241,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService } } - /** - * Build the configuration directory passed in or of the target FS - * @return the file - */ - public File getLocalConfDir() { - File confdir = - new File(SliderKeys.PROPAGATED_CONF_DIR_NAME).getAbsoluteFile(); - return confdir; - } - - /** - * Get the path to the DFS configuration that is defined in the cluster specification - * @return the generated configuration dir - */ - public String getGeneratedConfDir() { - return getGlobalInternalOptions().get( - InternalKeys.INTERNAL_GENERATED_CONF_PATH); - } - - /** - * Get the global internal options for the AM - * @return a map to access the internals - */ - public MapOperations getGlobalInternalOptions() { - return getInstanceDefinition() - .getInternalOperations(). - getGlobalOptions(); - } - /** * Get the filesystem of this cluster * @return the FS of the config @@ -1480,11 +1327,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService Exception exception = stopAction.getEx(); appStatus = stopAction.getFinalApplicationStatus(); - if (!spawnedProcessExitedBeforeShutdownTriggered) { - //stopped the forked process but don't worry about its exit code - int forkedExitCode = stopForkedProcess(); - log.debug("Stopped forked process: exit code={}", forkedExitCode); - } // make sure the AM is actually registered. If not, there's no point // trying to unregister it @@ -1500,7 +1342,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService launchService.stop(); //now release all containers - releaseAllContainers(); + releaseAllContainers(application); + DefaultMetricsSystem.shutdown(); // When the application completes, it should send a finish application // signal to the RM @@ -1536,7 +1379,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService /** * Start the slider RPC server */ - private void startSliderRPCServer(AggregateConf instanceDefinition) + private void startSliderRPCServer() throws IOException, SliderException { verifyIPCAccess(); @@ -1612,16 +1455,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService //for each assignment: instantiate that role for (ContainerAssignment assignment : assignments) { - try { - launchService.launchRole(assignment, getInstanceDefinition(), - buildContainerCredentials()); - } catch (IOException e) { - // Can be caused by failure to renew credentials with the remote - // service. If so, don't launch the application. Container is retained, - // though YARN will take it away after a timeout. - log.error("Failed to build credentials to launch container: {}", e, e); - - } + //TODO Do we need to pass credentials to containers? + launchService.launchRole(assignment, application, null); } //for all the operations, exec them @@ -1645,7 +1480,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // non complete containers should not be here assert (status.getState() == ContainerState.COMPLETE); - AppState.NodeCompletionResult result = appState.onCompletedNode(status); + AppState.NodeCompletionResult result = appState.onCompletedContainer(status); if (result.containerFailed) { RoleInstance ri = result.roleInstance; log.error("Role instance {} failed ", ri); @@ -1653,7 +1488,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService // known nodes trigger notifications if(!result.unknownNode) { - getProviderService().notifyContainerCompleted(containerId); queue(new UnregisterComponentInstance(containerId, 0, TimeUnit.MILLISECONDS)); } @@ -1724,22 +1558,14 @@ public class SliderAppMaster extends AbstractSliderLaunchedService * Implementation of cluster flexing. * It should be the only way that anything -even the AM itself on startup- * asks for nodes. - * @param resources the resource tree * @throws SliderException slider problems, including invalid configs * @throws IOException IO problems */ - public void flexCluster(ConfTree resources) + public void flexCluster(Messages.FlexComponentRequestProto request) throws IOException, SliderException { - - AggregateConf newConf = - new AggregateConf(appState.getInstanceDefinitionSnapshot()); - newConf.setResources(resources); - // verify the new definition is valid - sliderAMProvider.validateInstanceDefinition(newConf); - providerService.validateInstanceDefinition(newConf); - - appState.updateResourceDefinitions(resources); - + if (request != null) { + appState.updateComponents(request); + } // reset the scheduled windows...the values // may have changed appState.resetFailureCounts(); @@ -1750,24 +1576,37 @@ public class SliderAppMaster extends AbstractSliderLaunchedService /** * Schedule the failure window - * @param resources the resource tree * @throws BadConfigException if the window is out of range */ - private void scheduleFailureWindowResets(ConfTree resources) throws - BadConfigException { + private void scheduleFailureWindowResets( + org.apache.slider.api.resource.Configuration conf) { + ResetFailureWindow reset = new ResetFailureWindow(rmOperationHandler); - ConfTreeOperations ops = new ConfTreeOperations(resources); - MapOperations globals = ops.getGlobalOptions(); - long seconds = globals.getTimeRange(ResourceKeys.CONTAINER_FAILURE_WINDOW, - ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS, - ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS, - ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES, 0); - if (seconds > 0) { - log.info( - "Scheduling the failure window reset interval to every {} seconds", - seconds); - RenewingAction renew = new RenewingAction<>( - reset, seconds, seconds, TimeUnit.SECONDS, 0); + + long days = + conf.getPropertyLong(ResourceKeys.CONTAINER_FAILURE_WINDOW + ".days", + ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS); + long hours = + conf.getPropertyLong(ResourceKeys.CONTAINER_FAILURE_WINDOW + ".hours", + ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS); + long minutes = + conf.getPropertyLong(ResourceKeys.CONTAINER_FAILURE_WINDOW + ".minutes", + ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES); + long seconds = + conf.getPropertyLong(ResourceKeys.CONTAINER_FAILURE_WINDOW + ".seconds", + 0); + Preconditions + .checkState(days >= 0 && hours >= 0 && minutes >= 0 && seconds >= 0, + "Time range for has negative time component %s:%s:%s:%s", days, + hours, minutes, seconds); + long totalMinutes = days * 24 * 60 + hours * 24 + minutes; + long totalSeconds = totalMinutes * 60 + seconds; + if (totalSeconds > 0) { + log.info("Scheduling the failure window reset interval to every {}" + + " seconds", totalSeconds); + RenewingAction renew = + new RenewingAction<>(reset, totalSeconds, totalSeconds, + TimeUnit.SECONDS, 0); actionQueues.renewing("failures", renew); } else { log.info("Failure window reset interval is not set"); @@ -1776,16 +1615,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService /** * Schedule the escalation action - * @param internal * @throws BadConfigException */ - private void scheduleEscalation(ConfTree internal) throws BadConfigException { + private void scheduleEscalation( + org.apache.slider.api.resource.Configuration conf) { EscalateOutstandingRequests escalate = new EscalateOutstandingRequests(); - ConfTreeOperations ops = new ConfTreeOperations(internal); - int seconds = ops.getGlobalOptions().getOptionInt(InternalKeys.ESCALATION_CHECK_INTERVAL, + long seconds = conf.getPropertyLong(InternalKeys.ESCALATION_CHECK_INTERVAL, InternalKeys.DEFAULT_ESCALATION_CHECK_INTERVAL); - RenewingAction renew = new RenewingAction<>( - escalate, seconds, seconds, TimeUnit.SECONDS, 0); + RenewingAction renew = + new RenewingAction<>(escalate, seconds, seconds, TimeUnit.SECONDS, 0); actionQueues.renewing("escalation", renew); } @@ -1794,7 +1632,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService * @param reason reason for operation */ private synchronized void reviewRequestAndReleaseNodes(String reason) { - log.debug("reviewRequestAndReleaseNodes({})", reason); + log.info("reviewRequestAndReleaseNodes({})", reason); queue(new ReviewAndFlexApplicationSize(reason, 0, TimeUnit.SECONDS)); } @@ -1810,6 +1648,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService if ( actionQueues.hasQueuedActionWithAttribute( AsyncAction.ATTR_REVIEWS_APP_SIZE | AsyncAction.ATTR_HALTS_APP)) { + //TODO Loop all actions to check duplicate ？？ // this operation isn't needed at all -existing duplicate or shutdown due return; } @@ -1829,14 +1668,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService public synchronized void executeNodeReview(String reason) throws SliderInternalStateException { - log.debug("in executeNodeReview({})", reason); + log.info("in executeNodeReview({})", reason); if (amCompletionFlag.get()) { log.info("Ignoring node review operation: shutdown in progress"); } try { List allOperations = appState.reviewRequestAndReleaseNodes(); - // tell the provider - providerRMOperationHandler.execute(allOperations); //now apply the operations execute(allOperations); } catch (TriggerClusterTeardownException e) { @@ -1853,7 +1690,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService */ public void escalateOutstandingRequests() { List operations = appState.escalateOutstandingRequests(); - providerRMOperationHandler.execute(operations); execute(operations); } @@ -1861,11 +1697,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService /** * Shutdown operation: release all containers */ - private void releaseAllContainers() { + private void releaseAllContainers(Application application) { // Add the sleep here (before releasing containers) so that applications get // time to perform graceful shutdown try { - long timeout = getContainerReleaseTimeout(); + long timeout = getContainerReleaseTimeout(application); if (timeout > 0) { Thread.sleep(timeout); } @@ -1873,22 +1709,16 @@ public class SliderAppMaster extends AbstractSliderLaunchedService log.info("Sleep for container release interrupted"); } finally { List operations = appState.releaseAllContainers(); - providerRMOperationHandler.execute(operations); // now apply the operations execute(operations); } } - private long getContainerReleaseTimeout() { + private long getContainerReleaseTimeout(Application application) { // Get container release timeout in millis or 0 if the property is not set. - // If non-zero then add the agent heartbeat delay time, since it can take up - // to that much time for agents to receive the stop command. - int timeout = getInstanceDefinition().getAppConfOperations() - .getGlobalOptions() - .getOptionInt(SliderKeys.APP_CONTAINER_RELEASE_TIMEOUT, 0); - if (timeout > 0) { - timeout += SliderKeys.APP_CONTAINER_HEARTBEAT_INTERVAL_SEC; - } + long timeout = application.getConfiguration() + .getPropertyLong(SliderKeys.APP_CONTAINER_RELEASE_TIMEOUT, 0); + // convert to millis long timeoutInMillis = timeout * 1000l; log.info("Container release timeout in millis = {}", timeoutInMillis); @@ -2000,27 +1830,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService /** * Launch the provider service - * - * @param instanceDefinition definition of the service - * @param confDir directory of config data * @throws IOException * @throws SliderException */ - protected synchronized void launchProviderService(AggregateConf instanceDefinition, - File confDir) - throws IOException, SliderException { - Map env = new HashMap<>(); - boolean execStarted = providerService.exec(instanceDefinition, confDir, env, - this); - if (execStarted) { - providerService.registerServiceListener(this); - providerService.start(); - } else { - // didn't start, so don't register - providerService.start(); - // and send the started event ourselves - eventCallbackEvent(null); - } + protected synchronized void launchProviderService() + throws IOException, SliderException { + // didn't start, so don't register + providerService.start(); + // and send the started event ourselves + eventCallbackEvent(null); } /* =================================================================== */ @@ -2029,11 +1847,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService @Override // ProviderCompleted public void eventCallbackEvent(Object parameter) { - // signalled that the child process is up. - appState.noteAMLive(); // now ask for the cluster nodes try { - flexCluster(getInstanceDefinition().getResources()); + flexCluster(null); } catch (Exception e) { // cluster flex failure: log log.error("Failed to flex cluster nodes: {}", e, e); @@ -2064,62 +1880,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService } } - /* =================================================================== */ - /* ServiceStateChangeListener */ - /* =================================================================== */ - - /** - * Received on listening service termination. - * @param service the service that has changed. - */ - @Override //ServiceStateChangeListener - public void stateChanged(Service service) { - if (service == providerService && service.isInState(STATE.STOPPED)) { - //its the current master process in play - int exitCode = providerService.getExitCode(); - int mappedProcessExitCode = exitCode; - - boolean shouldTriggerFailure = !amCompletionFlag.get() - && (mappedProcessExitCode != 0); - - if (shouldTriggerFailure) { - String reason = - "Spawned process failed with raw " + exitCode + " mapped to " + - mappedProcessExitCode; - ActionStopSlider stop = new ActionStopSlider("stop", - mappedProcessExitCode, - FinalApplicationStatus.FAILED, - reason); - //this wasn't expected: the process finished early - spawnedProcessExitedBeforeShutdownTriggered = true; - log.info( - "Process has exited with exit code {} mapped to {} -triggering termination", - exitCode, - mappedProcessExitCode); - - //tell the AM the cluster is complete - signalAMComplete(stop); - } else { - //we don't care - log.info( - "Process has exited with exit code {} mapped to {} -ignoring", - exitCode, - mappedProcessExitCode); - } - } else { - super.stateChanged(service); - } - } - - /** - * stop forked process if it the running process var is not null - * @return the process exit code - */ - protected synchronized Integer stopForkedProcess() { - providerService.stop(); - return providerService.getExitCode(); - } - /** * Async start container request * @param container container @@ -2221,16 +1981,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService LOG_YARN.warn("Failed to stop Container {}", containerId); } - public AggregateConf getInstanceDefinition() { - return appState.getInstanceDefinition(); - } - - /** - * This is the status, the live model - */ - public ClusterDescription getClusterDescription() { - return appState.getClusterStatus(); - } public ProviderService getProviderService() { return providerService; @@ -2278,12 +2028,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService } /** - * Start the chaos monkey + * TODO Start the chaos monkey * @return true if it started */ private boolean maybeStartMonkey() { - MapOperations internals = getGlobalInternalOptions(); - +// MapOperations internals = getGlobalInternalOptions(); + MapOperations internals = new MapOperations(); Boolean enabled = internals.getOptionBool(InternalKeys.CHAOS_MONKEY_ENABLED, InternalKeys.DEFAULT_CHAOS_MONKEY_ENABLED); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java index 6b616816588..a660958b242 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java @@ -18,6 +18,7 @@ package org.apache.slider.server.appmaster.actions; +import org.apache.slider.api.proto.Messages; import org.apache.slider.core.conf.ConfTree; import org.apache.slider.server.appmaster.SliderAppMaster; import org.apache.slider.server.appmaster.state.AppState; @@ -26,19 +27,16 @@ import java.util.concurrent.TimeUnit; public class ActionFlexCluster extends AsyncAction { - public final ConfTree resources; - - public ActionFlexCluster(String name, - long delay, - TimeUnit timeUnit, ConfTree resources) { + final Messages.FlexComponentRequestProto requestProto; + public ActionFlexCluster(String name, long delay, TimeUnit timeUnit, + Messages.FlexComponentRequestProto requestProto) { super(name, delay, timeUnit, ATTR_CHANGES_APP_SIZE); - this.resources = resources; + this.requestProto = requestProto; } - @Override public void execute(SliderAppMaster appMaster, QueueAccess queueService, AppState appState) throws Exception { - appMaster.flexCluster(resources); + appMaster.flexCluster(requestProto); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java index ca330afb229..0d7f7d42319 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java @@ -19,6 +19,7 @@ package org.apache.slider.server.appmaster.actions; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.slider.api.resource.Application; import org.apache.slider.server.appmaster.SliderAppMaster; import org.apache.slider.server.appmaster.state.AppState; @@ -31,21 +32,13 @@ public class ActionRegisterServiceInstance extends AsyncAction { private final String instanceName; private final ApplicationId appId; - + private final Application application; public ActionRegisterServiceInstance(String instanceName, - ApplicationId appId) { + ApplicationId appId, Application application) { super("ActionRegisterServiceInstance"); this.instanceName = instanceName; this.appId = appId; - } - - public ActionRegisterServiceInstance(String instanceName, - ApplicationId appId, - long delay, - TimeUnit timeUnit) { - super("ActionRegisterServiceInstance", delay, timeUnit); - this.instanceName = instanceName; - this.appId = appId; + this.application = application; } @Override @@ -54,6 +47,6 @@ public class ActionRegisterServiceInstance extends AsyncAction { AppState appState) throws Exception { // YARN Registry do the registration - appMaster.registerServiceInstance(instanceName, appId); + appMaster.registerServiceInstance(instanceName, appId, application); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java new file mode 100644 index 00000000000..510ff73e9bf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/metrics/SliderMetrics.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.metrics; + +import com.codahale.metrics.Counter; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.annotation.Metrics; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MetricsRegistry; +import org.apache.hadoop.metrics2.lib.MutableGaugeInt; + +import java.util.HashMap; +import java.util.Map; + +import static org.apache.hadoop.metrics2.lib.Interns.info; + +@Metrics(context = "yarn-native-service") +public class SliderMetrics implements MetricsSource { + + @Metric("containers pending") + public MutableGaugeInt containersPending; + @Metric("anti-affinity containers pending") + public MutableGaugeInt pendingAAContainers; + @Metric("containers pending") + public MutableGaugeInt containersRunning; + @Metric("containers requested") + public MutableGaugeInt containersDesired; + @Metric("containers completed") + public MutableGaugeInt containersCompleted; + @Metric("containers failed") + public MutableGaugeInt containersFailed; + @Metric("containers failed since last threshold") + public MutableGaugeInt failedSinceLastThreshold; + @Metric("containers preempted") + public MutableGaugeInt containersPreempted; + @Metric("containers surplus") + public MutableGaugeInt surplusContainers; + + protected final MetricsRegistry registry; + + public SliderMetrics(MetricsInfo metricsInfo) { + registry = new MetricsRegistry(metricsInfo); + } + + @Override + public void getMetrics(MetricsCollector collector, boolean all) { + registry.snapshot(collector.addRecord(registry.info()), all); + } + + public static SliderMetrics register(String name, String description) { + SliderMetrics metrics = new SliderMetrics(info(name, description)); + DefaultMetricsSystem.instance().register(name, description, metrics); + return metrics; + } + + public void tag(String name, String description, String value) { + registry.tag(name, description, value); + } +} + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java deleted file mode 100644 index 972cc30a1d4..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.operations; - -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.slider.providers.ProviderService; - -import java.util.List; - -public class ProviderNotifyingOperationHandler extends RMOperationHandler { - - private final ProviderService providerService; - - public ProviderNotifyingOperationHandler(ProviderService providerService) { - this.providerService = providerService; - } - - @Override - public void releaseAssignedContainer(ContainerId containerId) { - providerService.releaseAssignedContainer(containerId); - } - - @Override - public void addContainerRequest(AMRMClient.ContainerRequest req) { - providerService.addContainerRequest(req); - } - - @Override - public int cancelContainerRequests(Priority priority1, - Priority priority2, - int count) { - return providerService.cancelContainerRequests(priority1, priority2, count); - } - - @Override - public void cancelSingleRequest(AMRMClient.ContainerRequest request) { - providerService.cancelSingleRequest(request); - } - - @Override - public void updateBlacklist(List blacklistAdditions, - List blacklistRemovals) { - providerService.updateBlacklist(blacklistAdditions, blacklistRemovals); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java index fbd408e6a69..4d483c79e82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java @@ -70,11 +70,12 @@ public class SliderClusterProtocolPBImpl implements SliderClusterProtocolPB { } @Override - public Messages.FlexClusterResponseProto flexCluster(RpcController controller, - Messages.FlexClusterRequestProto request) throws ServiceException { + public Messages.FlexComponentResponseProto flexComponent( + RpcController controller, Messages.FlexComponentRequestProto request) + throws ServiceException { try { - return real.flexCluster(request); - } catch (Exception e) { + return real.flexComponent(request); + } catch (IOException e) { throw wrap(e); } } @@ -90,19 +91,6 @@ public class SliderClusterProtocolPBImpl implements SliderClusterProtocolPB { } } - - @Override - public Messages.GetInstanceDefinitionResponseProto getInstanceDefinition( - RpcController controller, - Messages.GetInstanceDefinitionRequestProto request) - throws ServiceException { - try { - return real.getInstanceDefinition(request); - } catch (Exception e) { - throw wrap(e); - } - } - @Override public Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole( RpcController controller, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java index 448c6f338cb..c60d6090002 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java @@ -110,10 +110,10 @@ public class SliderClusterProtocolProxy implements SliderClusterProtocol { } @Override - public Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request) - throws IOException { + public Messages.FlexComponentResponseProto flexComponent( + Messages.FlexComponentRequestProto request) throws IOException { try { - return endpoint.flexCluster(NULL_CONTROLLER, request); + return endpoint.flexComponent(NULL_CONTROLLER, request); } catch (ServiceException e) { throw convert(e); } @@ -131,19 +131,6 @@ public class SliderClusterProtocolProxy implements SliderClusterProtocol { } } - - @Override - public Messages.GetInstanceDefinitionResponseProto getInstanceDefinition( - Messages.GetInstanceDefinitionRequestProto request) throws - IOException, - YarnException { - try { - return endpoint.getInstanceDefinition(NULL_CONTROLLER, request); - } catch (ServiceException e) { - throw convert(e); - } - } - @Override public Messages.ListNodeUUIDsByRoleResponseProto listNodeUUIDsByRole(Messages.ListNodeUUIDsByRoleRequestProto request) throws IOException, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java index 70c2f0584ab..344495b41b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java @@ -24,9 +24,9 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.SliderClusterProtocol; import org.apache.slider.api.proto.Messages; +import org.apache.slider.api.resource.Application; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.ContainerInformation; @@ -38,6 +38,7 @@ import org.apache.slider.core.exceptions.ServiceNotReadyException; import org.apache.slider.core.main.LauncherExitCodes; import org.apache.slider.core.persist.AggregateConfSerDeser; import org.apache.slider.core.persist.ConfTreeSerDeser; +import org.apache.slider.core.persist.JsonSerDeser; import org.apache.slider.server.appmaster.AppMasterActionOperations; import org.apache.slider.server.appmaster.actions.ActionFlexCluster; import org.apache.slider.server.appmaster.actions.ActionHalt; @@ -78,6 +79,9 @@ public class SliderIPCService extends AbstractService private final MetricsAndMonitoring metricsAndMonitoring; private final AppMasterActionOperations amOperations; private final ContentCache cache; + private static final JsonSerDeser jsonSerDeser = + new JsonSerDeser(Application.class); + /** * This is the prefix used for metrics @@ -195,17 +199,12 @@ public class SliderIPCService extends AbstractService return Messages.UpgradeContainersResponseProto.getDefaultInstance(); } - @Override //SliderClusterProtocol - public Messages.FlexClusterResponseProto flexCluster(Messages.FlexClusterRequestProto request) - throws IOException { + @Override + public Messages.FlexComponentResponseProto flexComponent( + Messages.FlexComponentRequestProto request) throws IOException { onRpcCall("flex"); - String payload = request.getClusterSpec(); - ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser(); - ConfTree updatedResources = confTreeSerDeser.fromJson(payload); - schedule(new ActionFlexCluster("flex", 1, TimeUnit.MILLISECONDS, - updatedResources)); - return Messages.FlexClusterResponseProto.newBuilder().setResponse( - true).build(); + schedule(new ActionFlexCluster("flex", 1, TimeUnit.MILLISECONDS, request)); + return Messages.FlexComponentResponseProto.newBuilder().build(); } @Override //SliderClusterProtocol @@ -216,38 +215,10 @@ public class SliderIPCService extends AbstractService String result; //quick update //query and json-ify - ClusterDescription cd = state.refreshClusterStatus(); - result = cd.toJsonString(); - String stat = result; + Application application = state.refreshClusterStatus(); + String stat = jsonSerDeser.toJson(application); return Messages.GetJSONClusterStatusResponseProto.newBuilder() - .setClusterSpec(stat) - .build(); - } - - @Override - public Messages.GetInstanceDefinitionResponseProto getInstanceDefinition( - Messages.GetInstanceDefinitionRequestProto request) - throws IOException, YarnException { - - onRpcCall("getinstancedefinition"); - String internal; - String resources; - String app; - AggregateConf instanceDefinition = - state.getInstanceDefinitionSnapshot(); - internal = instanceDefinition.getInternal().toJson(); - resources = instanceDefinition.getResources().toJson(); - app = instanceDefinition.getAppConf().toJson(); - assert internal != null; - assert resources != null; - assert app != null; - log.debug("Generating getInstanceDefinition Response"); - Messages.GetInstanceDefinitionResponseProto.Builder builder = - Messages.GetInstanceDefinitionResponseProto.newBuilder(); - builder.setInternal(internal); - builder.setResources(resources); - builder.setApplication(app); - return builder.build(); + .setClusterSpec(stat).build(); } @Override //SliderClusterProtocol diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java index 9a89c393a55..b31babc3436 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java @@ -138,6 +138,7 @@ public class SecurityConfiguration { public File getKeytabFile(AggregateConf instanceDefinition) throws SliderException, IOException { + //TODO implement this for dash semantic String keytabFullPath = instanceDefinition.getAppConfOperations() .getComponent(SliderKeys.COMPONENT_AM) .get(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java index 6f54959911b..e891a275652 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java @@ -18,12 +18,12 @@ package org.apache.slider.server.appmaster.state; -import com.codahale.metrics.Metric; -import com.codahale.metrics.MetricRegistry; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.metrics2.lib.MutableGaugeInt; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; @@ -31,42 +31,35 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.slider.api.ClusterDescription; -import org.apache.slider.api.ClusterDescriptionKeys; -import org.apache.slider.api.ClusterDescriptionOperations; import org.apache.slider.api.ClusterNode; import org.apache.slider.api.InternalKeys; -import org.apache.slider.api.ResourceKeys; import org.apache.slider.api.StatusKeys; +import org.apache.slider.api.proto.Messages; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.ApplicationState; +import org.apache.slider.api.resource.Component; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.RoleStatistics; import org.apache.slider.common.SliderExitCodes; import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.tools.ConfigHelper; import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTree; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.conf.MapOperations; import org.apache.slider.core.exceptions.BadClusterStateException; import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.core.exceptions.ErrorStrings; import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.exceptions.SliderInternalStateException; import org.apache.slider.core.exceptions.TriggerClusterTeardownException; -import org.apache.slider.core.persist.AggregateConfSerDeser; -import org.apache.slider.core.persist.ConfTreeSerDeser; import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; -import org.apache.slider.server.appmaster.management.LongGauge; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.management.MetricsConstants; +import org.apache.slider.server.appmaster.metrics.SliderMetrics; import org.apache.slider.server.appmaster.operations.AbstractRMOperation; import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation; import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; @@ -77,7 +70,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -89,12 +81,10 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicInteger; +import static org.apache.hadoop.metrics2.lib.Interns.info; import static org.apache.slider.api.ResourceKeys.*; -import static org.apache.slider.api.RoleKeys.*; import static org.apache.slider.api.StateValues.*; -import static org.apache.slider.providers.docker.DockerKeys.DEFAULT_DOCKER_USE_PRIVILEGED; -import static org.apache.slider.providers.docker.DockerKeys.DOCKER_IMAGE; -import static org.apache.slider.providers.docker.DockerKeys.DOCKER_USE_PRIVILEGED; +import static org.apache.slider.api.resource.ApplicationState.STARTED; /** * The model of all the ongoing state of a Slider AM. @@ -117,53 +107,8 @@ public class AppState { */ private boolean applicationLive = false; - /** - * The definition of the instance. Flexing updates the resources section - * This is used as a synchronization point on activities that update - * the CD, and also to update some of the structures that - * feed in to the CD - */ - private AggregateConf instanceDefinition; + private Application app; - /** - * Time the instance definition snapshots were created - */ - private long snapshotTime; - - /** - * Snapshot of the instance definition. This is fully - * resolved. - */ - private AggregateConf instanceDefinitionSnapshot; - - /** - * Snapshot of the raw instance definition; unresolved and - * without any patch of an AM into it. - */ - private AggregateConf unresolvedInstanceDefinition; - - /** - * snapshot of resources as of last update time - */ - private ConfTreeOperations resourcesSnapshot; - private ConfTreeOperations appConfSnapshot; - private ConfTreeOperations internalsSnapshot; - - /** - * This is the status, the live model - */ - private ClusterDescription clusterStatus = new ClusterDescription(); - - /** - * Metadata provided by the AM for use in filling in status requests - */ - private Map applicationInfo; - - /** - * Client properties created via the provider -static for the life - * of the application - */ - private Map clientProperties = new HashMap<>(); /** * This is a template of the cluster status @@ -179,11 +124,6 @@ public class AppState { private final ConcurrentSkipListMap rolePriorityMap = new ConcurrentSkipListMap<>(); - /** - * The master node. - */ - private RoleInstance appMasterNode; - /** * Hash map of the containers we have. This includes things that have * been allocated but are not live; it is a superset of the live list @@ -198,37 +138,6 @@ public class AppState { */ private final ConcurrentMap containersBeingReleased = new ConcurrentHashMap<>(); - - /** - * Counter for completed containers ( complete denotes successful or failed ) - */ - private final LongGauge completedContainerCount = new LongGauge(); - - /** - * Count of failed containers - */ - private final LongGauge failedContainerCount = new LongGauge(); - - /** - * # of started containers - */ - private final LongGauge startedContainers = new LongGauge(); - - /** - * # of containers that failed to start - */ - private final LongGauge startFailedContainerCount = new LongGauge(); - - /** - * Track the number of surplus containers received and discarded - */ - private final LongGauge surplusContainers = new LongGauge(); - - /** - * Track the number of requested containers. - * Important: this does not include AA requests which are yet to be issued. - */ - private final LongGauge outstandingContainerRequests = new LongGauge(); /** * Map of requested nodes. This records the command used to start it, @@ -256,7 +165,7 @@ public class AppState { * Nodes that came assigned to a role above that * which were asked for -this appears to happen */ - private final Set surplusNodes = new HashSet<>(); + private final Set surplusContainers = new HashSet<>(); /** * Map of containerID to cluster nodes, for status reports. @@ -269,7 +178,6 @@ public class AppState { private final AtomicInteger completionOfUnknownContainerEvent = new AtomicInteger(); - /** * limits of container core numbers in this queue */ @@ -298,6 +206,7 @@ public class AppState { private Resource minResource; private Resource maxResource; + private SliderMetrics appMetrics; /** * Create an instance * @param recordFactory factory for YARN records @@ -309,60 +218,6 @@ public class AppState { Preconditions.checkArgument(metricsAndMonitoring != null, "null metricsAndMonitoring"); this.recordFactory = recordFactory; this.metricsAndMonitoring = metricsAndMonitoring; - - // register any metrics - register(MetricsConstants.CONTAINERS_OUTSTANDING_REQUESTS, outstandingContainerRequests); - register(MetricsConstants.CONTAINERS_SURPLUS, surplusContainers); - register(MetricsConstants.CONTAINERS_STARTED, startedContainers); - register(MetricsConstants.CONTAINERS_COMPLETED, completedContainerCount); - register(MetricsConstants.CONTAINERS_FAILED, failedContainerCount); - register(MetricsConstants.CONTAINERS_START_FAILED, startFailedContainerCount); - } - - private void register(String name, Metric counter) { - this.metricsAndMonitoring.getMetrics().register( - MetricRegistry.name(AppState.class, name), counter); - } - - public long getFailedCountainerCount() { - return failedContainerCount.getCount(); - } - - /** - * Increment the count - */ - public void incFailedCountainerCount() { - failedContainerCount.inc(); - } - - public long getStartFailedCountainerCount() { - return startFailedContainerCount.getCount(); - } - - /** - * Increment the count and return the new value - */ - public void incStartedCountainerCount() { - startedContainers.inc(); - } - - public long getStartedCountainerCount() { - return startedContainers.getCount(); - } - - /** - * Increment the count and return the new value - */ - public void incStartFailedCountainerCount() { - startFailedContainerCount.inc(); - } - - public AtomicInteger getCompletionOfNodeNotInLiveListEvent() { - return completionOfNodeNotInLiveListEvent; - } - - public AtomicInteger getCompletionOfUnknownContainerEvent() { - return completionOfUnknownContainerEvent; } @@ -370,13 +225,7 @@ public class AppState { return roleStatusMap; } - protected Map getRoleMap() { - return roles; - } - public Map getRolePriorityMap() { - return rolePriorityMap; - } private Map getStartingContainers() { return startingContainers; @@ -396,47 +245,13 @@ public class AppState { /** * Get the current view of the cluster status. - *

- * Calls to {@link #refreshClusterStatus()} trigger a - * refresh of this field. - *

* This is read-only * to the extent that changes here do not trigger updates in the * application state. * @return the cluster status */ - public synchronized ClusterDescription getClusterStatus() { - return clusterStatus; - } - - @VisibleForTesting - protected synchronized void setClusterStatus(ClusterDescription clusterDesc) { - this.clusterStatus = clusterDesc; - } - - /** - * Set the instance definition -this also builds the (now obsolete) - * cluster specification from it. - * - * Important: this is for early binding and must not be used after the build - * operation is complete. - * @param definition initial definition - * @throws BadConfigException - */ - public synchronized void setInitialInstanceDefinition(AggregateConf definition) - throws BadConfigException, IOException { - log.debug("Setting initial instance definition"); - // snapshot the definition - AggregateConfSerDeser serDeser = new AggregateConfSerDeser(); - - unresolvedInstanceDefinition = serDeser.fromInstance(definition); - - this.instanceDefinition = serDeser.fromInstance(definition); - onInstanceDefinitionUpdated(); - } - - public synchronized AggregateConf getInstanceDefinition() { - return instanceDefinition; + public synchronized Application getClusterStatus() { + return app; } /** @@ -475,58 +290,27 @@ public class AppState { maxResource = recordFactory.newResource(containerMaxMemory, containerMaxCores); } - public ConfTreeOperations getResourcesSnapshot() { - return resourcesSnapshot; - } - - public ConfTreeOperations getAppConfSnapshot() { - return appConfSnapshot; - } - - public ConfTreeOperations getInternalsSnapshot() { - return internalsSnapshot; - } - public boolean isApplicationLive() { return applicationLive; } - public long getSnapshotTime() { - return snapshotTime; - } - - public synchronized AggregateConf getInstanceDefinitionSnapshot() { - return instanceDefinitionSnapshot; - } - - public AggregateConf getUnresolvedInstanceDefinition() { - return unresolvedInstanceDefinition; - } public synchronized void buildInstance(AppStateBindingInfo binding) throws BadClusterStateException, BadConfigException, IOException { binding.validate(); log.debug("Building application state"); - publishedProviderConf = binding.publishedProviderConf; - applicationInfo = binding.applicationInfo != null ? binding.applicationInfo - : new HashMap(); - - clientProperties = new HashMap<>(); containerReleaseSelector = binding.releaseSelector; - - Set confKeys = ConfigHelper.sortedConfigKeys(publishedProviderConf); - - // Add the -site configuration properties - for (String key : confKeys) { - String val = publishedProviderConf.get(key); - clientProperties.put(key, val); - } - // set the cluster specification (once its dependency the client properties // is out the way - setInitialInstanceDefinition(binding.instanceDefinition); + this.app = binding.application; + appMetrics = SliderMetrics.register(app.getName(), + "Metrics for service"); + appMetrics + .tag("type", "Metrics type [component or service]", "service"); + appMetrics + .tag("appId", "Application id for service", app.getId()); //build the initial role list List roleList = new ArrayList<>(binding.roles); @@ -534,51 +318,40 @@ public class AppState { buildRole(providerRole); } - ConfTreeOperations resources = instanceDefinition.getResourceOperations(); - - Set roleNames = resources.getComponentNames(); - for (String name : roleNames) { + int priority = 1; + for (Component component : app.getComponents()) { + String name = component.getName(); if (roles.containsKey(name)) { continue; } - if (hasUniqueNames(resources, name)) { - log.info("Skipping group {}", name); + if (component.getUniqueComponentSupport()) { + log.info("Skipping group " + name + ", as it's unique component"); continue; } - // this is a new value - log.info("Adding role {}", name); - MapOperations resComponent = resources.getComponent(name); - ProviderRole dynamicRole = createDynamicProviderRole(name, resComponent); + log.info("Adding component: " + name); + ProviderRole dynamicRole = + createComponent(name, name, component, priority++); buildRole(dynamicRole); roleList.add(dynamicRole); } //then pick up the requirements buildRoleRequirementsFromResources(); - //set the livespan - MapOperations globalResOpts = instanceDefinition.getResourceOperations().getGlobalOptions(); - - startTimeThreshold = globalResOpts.getOptionInt( - InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE, - InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE); - - failureThreshold = globalResOpts.getOptionInt( - CONTAINER_FAILURE_THRESHOLD, + org.apache.slider.api.resource.Configuration conf = app.getConfiguration(); + startTimeThreshold = + conf.getPropertyLong(InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE, + InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE); + failureThreshold = (int) conf.getPropertyLong(CONTAINER_FAILURE_THRESHOLD, DEFAULT_CONTAINER_FAILURE_THRESHOLD); - nodeFailureThreshold = globalResOpts.getOptionInt( - NODE_FAILURE_THRESHOLD, + nodeFailureThreshold = (int) conf.getPropertyLong(NODE_FAILURE_THRESHOLD, DEFAULT_NODE_FAILURE_THRESHOLD); - initClusterStatus(); - // set up the role history roleHistory = new RoleHistory(roleStatusMap.values(), recordFactory); - roleHistory.register(metricsAndMonitoring); roleHistory.onStart(binding.fs, binding.historyPath); // trigger first node update roleHistory.onNodesUpdated(binding.nodeReports); - //rebuild any live containers rebuildModelFromRestart(binding.liveContainers); @@ -586,180 +359,57 @@ public class AppState { logServerURL = binding.serviceConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL, ""); //mark as live applicationLive = true; + app.setState(STARTED); } - public void initClusterStatus() { - //copy into cluster status. - ClusterDescription status = ClusterDescription.copy(clusterStatusTemplate); - status.state = STATE_CREATED; - MapOperations infoOps = new MapOperations("info", status.info); - infoOps.mergeWithoutOverwrite(applicationInfo); - SliderUtils.addBuildInfo(infoOps, "status"); + //TODO WHY do we need to create the component for AM ? + public ProviderRole createComponent(String name, String group, + Component component, int priority) throws BadConfigException { - long now = now(); - status.setInfoTime(StatusKeys.INFO_LIVE_TIME_HUMAN, - StatusKeys.INFO_LIVE_TIME_MILLIS, - now); - SliderUtils.setInfoTime(infoOps, - StatusKeys.INFO_LIVE_TIME_HUMAN, - StatusKeys.INFO_LIVE_TIME_MILLIS, - now); - if (0 == status.createTime) { - status.createTime = now; - SliderUtils.setInfoTime(infoOps, - StatusKeys.INFO_CREATE_TIME_HUMAN, - StatusKeys.INFO_CREATE_TIME_MILLIS, - now); - } - status.state = STATE_LIVE; + org.apache.slider.api.resource.Configuration conf = + component.getConfiguration(); + long placementTimeout = conf.getPropertyLong(PLACEMENT_ESCALATE_DELAY, + DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS); + long placementPolicy = conf.getPropertyLong(COMPONENT_PLACEMENT_POLICY, + PlacementPolicy.DEFAULT); + int threshold = (int) conf + .getPropertyLong(NODE_FAILURE_THRESHOLD, nodeFailureThreshold); + ProviderRole newRole = + new ProviderRole(name, group, priority, (int)placementPolicy, threshold, + placementTimeout, "", component); - //set the app state to this status - setClusterStatus(status); - } - - /** - * Build a dynamic provider role - * @param name name of role - * @return a new provider role - * @throws BadConfigException bad configuration - */ - public ProviderRole createDynamicProviderRole(String name, MapOperations component) - throws BadConfigException { - return createDynamicProviderRole(name, name, component); - } - - /** - * Build a dynamic provider role - * @param name name of role - * @param group group of role - * @return a new provider role - * @throws BadConfigException bad configuration - */ - public ProviderRole createDynamicProviderRole(String name, String group, MapOperations component) - throws BadConfigException { - String priOpt = component.getMandatoryOption(COMPONENT_PRIORITY); - int priority = SliderUtils.parseAndValidate( - "value of " + name + " " + COMPONENT_PRIORITY, priOpt, 0, 1, -1); - - String placementOpt = component.getOption(COMPONENT_PLACEMENT_POLICY, - Integer.toString(PlacementPolicy.DEFAULT)); - - int placement = SliderUtils.parseAndValidate( - "value of " + name + " " + COMPONENT_PLACEMENT_POLICY, placementOpt, 0, 0, -1); - - int placementTimeout = component.getOptionInt(PLACEMENT_ESCALATE_DELAY, - DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS); - - ProviderRole newRole = new ProviderRole(name, - group, - priority, - placement, - getNodeFailureThresholdForRole(group), - placementTimeout, - component.getOption(YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION)); - log.info("New {} ", newRole); + log.info("Created a new role " + newRole); return newRole; } - /** - * Actions to perform when an instance definition is updated - * Currently: - *

    - *
  1. - * resolve the configuration - *
    2. - *
  2. - * update the cluster spec derivative - *
    3. - *
- * - * @throws BadConfigException - */ - private synchronized void onInstanceDefinitionUpdated() - throws BadConfigException, IOException { - - log.debug("Instance definition updated"); - //note the time - snapshotTime = now(); - - for (String component : instanceDefinition.getResourceOperations().getComponentNames()) { - instanceDefinition.getAppConfOperations().getOrAddComponent(component); + public synchronized void updateComponents( + Messages.FlexComponentRequestProto requestProto) + throws BadConfigException { + for (Component component : app.getComponents()) { + if (component.getName().equals(requestProto.getName())) { + component + .setNumberOfContainers((long) requestProto.getNumberOfContainers()); + } } - - // resolve references if not already done - instanceDefinition.resolve(); - - // force in the AM desired state values - ConfTreeOperations resources = instanceDefinition.getResourceOperations(); - - if (resources.getComponent(SliderKeys.COMPONENT_AM) != null) { - resources.setComponentOpt( - SliderKeys.COMPONENT_AM, COMPONENT_INSTANCES, "1"); - } - - - //snapshot all three sectons - resourcesSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getResources()); - appConfSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getAppConf()); - internalsSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getInternal()); - //build a new aggregate from the snapshots - instanceDefinitionSnapshot = new AggregateConf(resourcesSnapshot.confTree, - appConfSnapshot.confTree, - internalsSnapshot.confTree); - instanceDefinitionSnapshot.setName(instanceDefinition.getName()); - - clusterStatusTemplate = ClusterDescriptionOperations.buildFromInstanceDefinition( - instanceDefinition); - - // Add the -site configuration properties - for (Map.Entry prop : clientProperties.entrySet()) { - clusterStatusTemplate.clientProperties.put(prop.getKey(), prop.getValue()); - } - - } - - /** - * The resource configuration is updated -review and update state. - * @param resources updated resources specification - * @return a list of any dynamically added provider roles - * (purely for testing purposes) - */ - @VisibleForTesting - public synchronized List updateResourceDefinitions(ConfTree resources) - throws BadConfigException, IOException { - log.debug("Updating resources to {}", resources); - // snapshot the (possibly unresolved) values - ConfTreeSerDeser serDeser = new ConfTreeSerDeser(); - unresolvedInstanceDefinition.setResources( - serDeser.fromInstance(resources)); - // assign another copy under the instance definition for resolving - // and then driving application size - instanceDefinition.setResources(serDeser.fromInstance(resources)); - onInstanceDefinitionUpdated(); - - // propagate the role table - Map> updated = resources.components; - getClusterStatus().roles = SliderUtils.deepClone(updated); - getClusterStatus().updateTime = now(); - return buildRoleRequirementsFromResources(); + //TODO update cluster description + buildRoleRequirementsFromResources(); } /** * build the role requirements from the cluster specification * @return a list of any dynamically added provider roles */ - private List buildRoleRequirementsFromResources() throws BadConfigException { + private List buildRoleRequirementsFromResources() + throws BadConfigException { List newRoles = new ArrayList<>(0); // now update every role's desired count. // if there are no instance values, that role count goes to zero - - ConfTreeOperations resources = - instanceDefinition.getResourceOperations(); - // Add all the existing roles + // component name -> number of containers Map groupCounts = new HashMap<>(); + for (RoleStatus roleStatus : getRoleStatusMap().values()) { if (roleStatus.isExcludeFromFlexing()) { // skip inflexible roles, e.g AM itself @@ -768,10 +418,11 @@ public class AppState { long currentDesired = roleStatus.getDesired(); String role = roleStatus.getName(); String roleGroup = roleStatus.getGroup(); - int desiredInstanceCount = getDesiredInstanceCount(resources, roleGroup); + Component component = roleStatus.getProviderRole().component; + int desiredInstanceCount = component.getNumberOfContainers().intValue(); int newDesired = desiredInstanceCount; - if (hasUniqueNames(resources, roleGroup)) { + if (component.getUniqueComponentSupport()) { Integer groupCount = 0; if (groupCounts.containsKey(roleGroup)) { groupCount = groupCounts.get(roleGroup); @@ -793,56 +444,54 @@ public class AppState { if (currentDesired != newDesired) { log.info("Role {} flexed from {} to {}", role, currentDesired, newDesired); - roleStatus.setDesired(newDesired); + setDesiredContainers(roleStatus, newDesired); } } // now the dynamic ones. Iterate through the the cluster spec and // add any role status entries not in the role status - Set roleNames = resources.getComponentNames(); - for (String name : roleNames) { + + List list = new ArrayList<>(getRoleStatusMap().values()); + for (RoleStatus roleStatus : list) { + String name = roleStatus.getName(); + Component component = roleStatus.getProviderRole().component; if (roles.containsKey(name)) { continue; } - if (hasUniqueNames(resources, name)) { + if (component.getUniqueComponentSupport()) { // THIS NAME IS A GROUP - int desiredInstanceCount = getDesiredInstanceCount(resources, name); + int desiredInstanceCount = component.getNumberOfContainers().intValue(); Integer groupCount = 0; if (groupCounts.containsKey(name)) { groupCount = groupCounts.get(name); } for (int i = groupCount + 1; i <= desiredInstanceCount; i++) { - int priority = resources.getComponentOptInt(name, COMPONENT_PRIORITY, i); + int priority = roleStatus.getPriority(); // this is a new instance of an existing group String newName = String.format("%s%d", name, i); int newPriority = getNewPriority(priority + i - 1); log.info("Adding new role {}", newName); - MapOperations component = resources.getComponent(name, - Collections.singletonMap(COMPONENT_PRIORITY, - Integer.toString(newPriority))); - if (component == null) { - throw new BadConfigException("Component is null for name = " + name - + ", newPriority =" + newPriority); - } - ProviderRole dynamicRole = createDynamicProviderRole(newName, name, component); - RoleStatus roleStatus = buildRole(dynamicRole); - roleStatus.setDesired(1); - log.info("New role {}", roleStatus); + ProviderRole dynamicRole = + createComponent(newName, name, component, newPriority); + RoleStatus newRole = buildRole(dynamicRole); + incDesiredContainers(newRole); + log.info("New role {}", newRole); if (roleHistory != null) { - roleHistory.addNewRole(roleStatus); + roleHistory.addNewRole(newRole); } newRoles.add(dynamicRole); } } else { // this is a new value log.info("Adding new role {}", name); - MapOperations component = resources.getComponent(name); - ProviderRole dynamicRole = createDynamicProviderRole(name, component); - RoleStatus roleStatus = buildRole(dynamicRole); - roleStatus.setDesired(getDesiredInstanceCount(resources, name)); - log.info("New role {}", roleStatus); + ProviderRole dynamicRole = + createComponent(name, name, component, roleStatus.getPriority()); + RoleStatus newRole = buildRole(dynamicRole); + incDesiredContainers(roleStatus, + component.getNumberOfContainers().intValue()); + log.info("New role {}", newRole); if (roleHistory != null) { - roleHistory.addNewRole(roleStatus); + roleHistory.addNewRole(newRole); } newRoles.add(dynamicRole); } @@ -860,37 +509,6 @@ public class AppState { return rolePriorityMap.lastKey() + 1; } - /** - * Get the desired instance count of a role, rejecting negative values - * @param resources resource map - * @param roleGroup role group - * @return the instance count - * @throws BadConfigException if the count is negative - */ - private int getDesiredInstanceCount(ConfTreeOperations resources, - String roleGroup) throws BadConfigException { - int desiredInstanceCount = - resources.getComponentOptInt(roleGroup, COMPONENT_INSTANCES, 0); - - if (desiredInstanceCount < 0) { - log.error("Role {} has negative desired instances : {}", roleGroup, - desiredInstanceCount); - throw new BadConfigException( - "Negative instance count (%) requested for component %s", - desiredInstanceCount, roleGroup); - } - return desiredInstanceCount; - } - - private Boolean hasUniqueNames(ConfTreeOperations resources, String group) { - MapOperations component = resources.getComponent(group); - if (component == null) { - log.info("Component was null for {} when checking unique names", group); - return Boolean.FALSE; - } - return component.getOptionBool(UNIQUE_NAMES, Boolean.FALSE); - } - /** * Add knowledge of a role. * This is a build-time operation that is not synchronized, and @@ -923,66 +541,9 @@ public class AppState { */ private void buildRoleResourceRequirements() { for (RoleStatus role : roleStatusMap.values()) { - role.setResourceRequirements( - buildResourceRequirements(role, recordFactory.newResource())); + role.setResourceRequirements(buildResourceRequirements(role)); } } - - /** - * build up the special master node, which lives - * in the live node set but has a lifecycle bonded to the AM - * @param containerId the AM master - * @param host hostname - * @param amPort port - * @param nodeHttpAddress http address: may be null - */ - public void buildAppMasterNode(ContainerId containerId, - String host, - int amPort, - String nodeHttpAddress) { - Container container = new ContainerPBImpl(); - container.setId(containerId); - NodeId nodeId = NodeId.newInstance(host, amPort); - container.setNodeId(nodeId); - container.setNodeHttpAddress(nodeHttpAddress); - RoleInstance am = new RoleInstance(container); - am.role = SliderKeys.COMPONENT_AM; - am.group = SliderKeys.COMPONENT_AM; - am.roleId = SliderKeys.ROLE_AM_PRIORITY_INDEX; - am.createTime =now(); - am.startTime = am.createTime; - appMasterNode = am; - //it is also added to the set of live nodes - getLiveContainers().put(containerId, am); - putOwnedContainer(containerId, am); - - // patch up the role status - RoleStatus roleStatus = roleStatusMap.get(SliderKeys.ROLE_AM_PRIORITY_INDEX); - roleStatus.setDesired(1); - roleStatus.incActual(); - roleStatus.incStarted(); - } - - /** - * Note that the master node has been launched, - * though it isn't considered live until any forked - * processes are running. It is NOT registered with - * the role history -the container is incomplete - * and it will just cause confusion - */ - public void noteAMLaunched() { - getLiveContainers().put(appMasterNode.getContainerId(), appMasterNode); - } - - /** - * AM declares ourselves live in the cluster description. - * This is meant to be triggered from the callback - * indicating the spawned process is up and running. - */ - public void noteAMLive() { - appMasterNode.state = STATE_LIVE; - } - /** * Look up the status entry of a role or raise an exception * @param key role ID @@ -1008,24 +569,6 @@ public class AppState { return lookupRoleStatus(ContainerPriority.extractRole(c)); } - /** - * Get a deep clone of the role status list. Concurrent events may mean this - * list (or indeed, some of the role status entries) may be inconsistent - * @return a snapshot of the role status entries - */ - public List cloneRoleStatusList() { - Collection statuses = roleStatusMap.values(); - List statusList = new ArrayList<>(statuses.size()); - try { - for (RoleStatus status : statuses) { - statusList.add((RoleStatus)(status.clone())); - } - } catch (CloneNotSupportedException e) { - log.warn("Unexpected cloning failure: {}", e, e); - } - return statusList; - } - /** * Look up a role in the map @@ -1278,8 +821,6 @@ public class AppState { } instance.released = true; containersBeingReleased.put(id, instance.container); - RoleStatus role = lookupRoleStatus(instance.roleId); - role.incReleasing(); roleHistory.onContainerReleaseSubmitted(container); } @@ -1292,10 +833,10 @@ public class AppState { * @return the container request to submit or null if there is none */ private AMRMClient.ContainerRequest createContainerRequest(RoleStatus role) { + incPendingContainers(role); if (role.isAntiAffinePlacement()) { return createAAContainerRequest(role); } else { - incrementRequestCount(role); OutstandingRequest request = roleHistory.requestContainerForRole(role); if (request != null) { return request.getIssuedRequest(); @@ -1318,96 +859,98 @@ public class AppState { if (request == null) { return null; } - incrementRequestCount(role); role.setOutstandingAArequest(request); return request.getIssuedRequest(); } - /** - * Increment the request count of a role. - *

- * Also updates application state counters - * @param role role being requested. - */ - protected void incrementRequestCount(RoleStatus role) { - role.incRequested(); - incOutstandingContainerRequests(); + private void incPendingContainers(RoleStatus role) { + role.getComponentMetrics().containersPending.incr(); + appMetrics.containersPending.incr(); } - /** - * Inc #of outstanding requests. - */ - private void incOutstandingContainerRequests() { - outstandingContainerRequests.inc(); + private void decPendingContainers(RoleStatus role) { + decPendingContainers(role, 1); } - /** - * Decrement the number of outstanding requests. This never goes below zero. - */ - private void decOutstandingContainerRequests() { - synchronized (outstandingContainerRequests) { - if (outstandingContainerRequests.getCount() > 0) { - // decrement but never go below zero - outstandingContainerRequests.dec(); - } + private void decPendingContainers(RoleStatus role, int n) { + role.getComponentMetrics().containersPending.decr(n);; + appMetrics.containersPending.decr(n); + } + + + private void incRunningContainers(RoleStatus role) { + role.getComponentMetrics().containersRunning.incr();; + appMetrics.containersRunning.incr(); + } + + private void decRunningContainers(RoleStatus role) { + role.getComponentMetrics().containersRunning.decr();; + appMetrics.containersRunning.decr(); + } + + private void setDesiredContainers(RoleStatus role, int n) { + role.getComponentMetrics().containersDesired.set(n); + appMetrics.containersDesired.set(n); + } + + private void incDesiredContainers(RoleStatus role) { + role.getComponentMetrics().containersDesired.incr(); + appMetrics.containersDesired.incr(); + } + + private void incDesiredContainers(RoleStatus role, int n) { + role.getComponentMetrics().containersDesired.incr(n); + appMetrics.containersDesired.incr(n); + } + + private void incCompletedContainers(RoleStatus role) { + role.getComponentMetrics().containersCompleted.incr(); + appMetrics.containersCompleted.incr(); + } + + private void incFailedContainers(RoleStatus role, ContainerOutcome outcome) { + role.getComponentMetrics().containersFailed.incr(); + appMetrics.containersFailed.incr(); + switch (outcome) { + case Preempted: + appMetrics.containersPreempted.incr(); + role.getComponentMetrics().containersPreempted.incr(); + break; + case Failed: + appMetrics.failedSinceLastThreshold.incr(); + break; + default: + break; } } - - /** - * Get the value of a YARN requirement (cores, RAM, etc). - * These are returned as integers, but there is special handling of the - * string {@link ResourceKeys#YARN_RESOURCE_MAX}, which triggers - * the return of the maximum value. - * @param group component to get from - * @param option option name - * @param defVal default value - * @param maxVal value to return if the max val is requested - * @return parsed value - * @throws NumberFormatException if the role could not be parsed. - */ - private int getResourceRequirement(ConfTreeOperations resources, - String group, - String option, - int defVal, - int maxVal) { - - String val = resources.getComponentOpt(group, option, - Integer.toString(defVal)); - Integer intVal; - if (YARN_RESOURCE_MAX.equals(val)) { - intVal = maxVal; - } else { - intVal = Integer.decode(val); - } - return intVal; - } - /** * Build up the resource requirements for this role from the * cluster specification, including substituing max allowed values * if the specification asked for it. * @param role role - * @param capability capability to set up. A new one may be created * during normalization */ - public Resource buildResourceRequirements(RoleStatus role, Resource capability) { + public Resource buildResourceRequirements(RoleStatus role) { // Set up resource requirements from role values String name = role.getName(); - String group = role.getGroup(); - ConfTreeOperations resources = getResourcesSnapshot(); - int cores = getResourceRequirement(resources, - group, - YARN_CORES, - DEF_YARN_CORES, - containerMaxCores); - capability.setVirtualCores(cores); - int ram = getResourceRequirement(resources, group, - YARN_MEMORY, - DEF_YARN_MEMORY, - containerMaxMemory); - capability.setMemory(ram); - log.debug("Component {} has RAM={}, vCores ={}", name, ram, cores); + Component component = role.getProviderRole().component; + if (component == null) { + // this is for AM container + // TODO why do we need to create the component for AM ? + return Resource.newInstance(1, 512); + } + int cores = Math.min(containerMaxCores, component.getResource().getCpus()); + if (cores <= 0) { + cores = DEF_YARN_CORES; + } + long mem = Math.min(containerMaxMemory, + Long.parseLong(component.getResource().getMemory())); + if (mem <= 0) { + mem = DEF_YARN_MEMORY; + } + Resource capability = Resource.newInstance(mem, cores); + log.debug("Component {} has RAM={}, vCores ={}", name, mem, cores); Resource normalized = recordFactory.normalize(capability, minResource, maxResource); if (!Resources.equals(normalized, capability)) { @@ -1459,7 +1002,6 @@ public class AppState { */ @VisibleForTesting public RoleInstance innerOnNodeManagerContainerStarted(ContainerId containerId) { - incStartedCountainerCount(); RoleInstance instance = getOwnedContainer(containerId); if (instance == null) { //serious problem @@ -1477,8 +1019,6 @@ public class AppState { "Container "+ containerId +" is already started"); } instance.state = STATE_LIVE; - RoleStatus roleStatus = lookupRoleStatus(instance.roleId); - roleStatus.incStarted(); Container container = instance.container; addLaunchedContainer(container, instance); return instance; @@ -1497,8 +1037,6 @@ public class AppState { public synchronized void onNodeManagerContainerStartFailed(ContainerId containerId, Throwable thrown) { removeOwnedContainer(containerId); - incFailedCountainerCount(); - incStartFailedCountainerCount(); RoleInstance instance = getStartingContainers().remove(containerId); if (null != instance) { RoleStatus roleStatus = lookupRoleStatus(instance.roleId); @@ -1509,9 +1047,10 @@ public class AppState { text = "container start failure"; } instance.diagnostics = text; - roleStatus.noteFailed(true, text, ContainerOutcome.Failed); + roleStatus.noteFailed(text); getFailedContainers().put(containerId, instance); roleHistory.onNodeManagerContainerStartFailed(instance.container); + incFailedContainers(roleStatus, ContainerOutcome.Failed); } } @@ -1607,7 +1146,8 @@ public class AppState { * @param status the node that has just completed * @return NodeCompletionResult */ - public synchronized NodeCompletionResult onCompletedNode(ContainerStatus status) { + public synchronized NodeCompletionResult onCompletedContainer( + ContainerStatus status) { ContainerId containerId = status.getContainerId(); NodeCompletionResult result = new NodeCompletionResult(); RoleInstance roleInstance; @@ -1618,18 +1158,16 @@ public class AppState { log.info("Container was queued for release : {}", containerId); Container container = containersBeingReleased.remove(containerId); RoleStatus roleStatus = lookupRoleStatus(container); - long releasing = roleStatus.decReleasing(); - long actual = roleStatus.decActual(); - long completedCount = roleStatus.incCompleted(); - log.info("decrementing role count for role {} to {}; releasing={}, completed={}", + decRunningContainers(roleStatus); + incCompletedContainers(roleStatus); + log.info("decrementing role count for role {} to {}; completed={}", roleStatus.getName(), - actual, - releasing, - completedCount); + roleStatus.getComponentMetrics().containersRunning.value(), + roleStatus.getComponentMetrics().containersCompleted.value()); result.outcome = ContainerOutcome.Completed; roleHistory.onReleaseCompleted(container); - } else if (surplusNodes.remove(containerId)) { + } else if (surplusContainers.remove(containerId)) { //its a surplus one being purged result.surplusNode = true; } else { @@ -1640,8 +1178,8 @@ public class AppState { roleInstance = removeOwnedContainer(containerId); if (roleInstance != null) { - //it was active, move it to failed - incFailedCountainerCount(); + RoleStatus roleStatus = lookupRoleStatus(roleInstance.roleId); + incFailedContainers(roleStatus, result.outcome); failedContainers.put(containerId, roleInstance); } else { // the container may have been noted as failed already, so look @@ -1653,8 +1191,8 @@ public class AppState { String rolename = roleInstance.role; log.info("Failed container in role[{}] : {}", roleId, rolename); try { - RoleStatus roleStatus = lookupRoleStatus(roleId); - roleStatus.decActual(); + RoleStatus roleStatus = lookupRoleStatus(roleInstance.roleId); + decRunningContainers(roleStatus); boolean shortLived = isShortLived(roleInstance); String message; Container failedContainer = roleInstance.container; @@ -1670,8 +1208,10 @@ public class AppState { } else { message = String.format("Failure %s (%d)", containerId, exitStatus); } - roleStatus.noteFailed(shortLived, message, result.outcome); - long failed = roleStatus.getFailed(); + roleStatus.noteFailed(message); + incFailedContainers(roleStatus, result.outcome); + long failed = + roleStatus.getComponentMetrics().containersFailed.value(); log.info("Current count of failed role[{}] {} = {}", roleId, rolename, failed); if (failedContainer != null) { @@ -1761,7 +1301,7 @@ public class AppState { float actual = 0; for (RoleStatus role : getRoleStatusMap().values()) { desired += role.getDesired(); - actual += role.getActual(); + actual += role.getRunning(); } if (desired == 0) { percentage = 100; @@ -1771,29 +1311,26 @@ public class AppState { return percentage; } + /** * Update the cluster description with the current application state */ - public ClusterDescription refreshClusterStatus() { - return refreshClusterStatus(null); - } + public synchronized Application refreshClusterStatus() { + + //TODO replace ClusterDescription with Application + related statistics + //TODO build container stats + app.setState(ApplicationState.STARTED); + return app; +/* + return app; - /** - * Update the cluster description with the current application state - * @param providerStatus status from the provider for the cluster info section - */ - public synchronized ClusterDescription refreshClusterStatus(Map providerStatus) { ClusterDescription cd = getClusterStatus(); long now = now(); cd.setInfoTime(StatusKeys.INFO_STATUS_TIME_HUMAN, StatusKeys.INFO_STATUS_TIME_MILLIS, now); - if (providerStatus != null) { - for (Map.Entry entry : providerStatus.entrySet()) { - cd.setInfo(entry.getKey(), entry.getValue()); - } - } + MapOperations infoOps = new MapOperations("info", cd.info); infoOps.mergeWithoutOverwrite(applicationInfo); SliderUtils.addBuildInfo(infoOps, "status"); @@ -1810,32 +1347,8 @@ public class AppState { cd.status = new HashMap<>(); cd.status.put(ClusterDescriptionKeys.KEY_CLUSTER_LIVE, clusterNodes); - for (RoleStatus role : getRoleStatusMap().values()) { String rolename = role.getName(); - if (hasUniqueNames(instanceDefinition.getResourceOperations(), - role.getGroup())) { - cd.setRoleOpt(rolename, COMPONENT_PRIORITY, role.getPriority()); - cd.setRoleOpt(rolename, ROLE_GROUP, role.getGroup()); - MapOperations groupOptions = instanceDefinition.getResourceOperations() - .getComponent(role.getGroup()); - SliderUtils.mergeMapsIgnoreDuplicateKeys(cd.getRole(rolename), - groupOptions.options); - } - String prefix = instanceDefinition.getAppConfOperations() - .getComponentOpt(role.getGroup(), ROLE_PREFIX, null); - if (SliderUtils.isSet(prefix)) { - cd.setRoleOpt(rolename, ROLE_PREFIX, SliderUtils.trimPrefix(prefix)); - } - String dockerImage = instanceDefinition.getAppConfOperations() - .getComponentOpt(role.getGroup(), DOCKER_IMAGE, null); - if (SliderUtils.isSet(dockerImage)) { - cd.setRoleOpt(rolename, DOCKER_IMAGE, dockerImage); - Boolean dockerUsePrivileged = instanceDefinition.getAppConfOperations() - .getComponentOptBool(role.getGroup(), DOCKER_USE_PRIVILEGED, - DEFAULT_DOCKER_USE_PRIVILEGED); - cd.setRoleOpt(rolename, DOCKER_USE_PRIVILEGED, dockerUsePrivileged); - } List instances = instanceMap.get(rolename); int nodeCount = instances != null ? instances.size(): 0; cd.setRoleOpt(rolename, COMPONENT_INSTANCES, @@ -1861,7 +1374,7 @@ public class AppState { // liveness cd.liveness = getApplicationLivenessInformation(); - return cd; + return cd;*/ } /** @@ -1878,29 +1391,6 @@ public class AppState { return li; } - /** - * Get the live statistics map - * @return a map of statistics values, defined in the {@link StatusKeys} - * keylist. - */ - protected Map getLiveStatistics() { - Map sliderstats = new HashMap<>(); - sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE, - liveNodes.size()); - sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED, - completedContainerCount.intValue()); - sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED, - failedContainerCount.intValue()); - sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED, - startedContainers.intValue()); - sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED, - startFailedContainerCount.intValue()); - sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_SURPLUS, - surplusContainers.intValue()); - sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_UNKNOWN_COMPLETED, - completionOfUnknownContainerEvent.get()); - return sliderstats; - } /** * Get the aggregate statistics across all roles @@ -1949,7 +1439,7 @@ public class AppState { */ public synchronized List reviewRequestAndReleaseNodes() throws SliderInternalStateException, TriggerClusterTeardownException { - log.debug("in reviewRequestAndReleaseNodes()"); + log.info("in reviewRequestAndReleaseNodes()"); List allOperations = new ArrayList<>(); AbstractRMOperation blacklistOperation = updateBlacklist(); if (blacklistOperation != null) { @@ -1981,15 +1471,11 @@ public class AppState { if (failures > threshold) { throw new TriggerClusterTeardownException( - SliderExitCodes.EXIT_DEPLOYMENT_FAILED, - FinalApplicationStatus.FAILED, ErrorStrings.E_UNSTABLE_CLUSTER + - " - failed with component %s failed 'recently' %d times (%d in startup);" + - " threshold is %d - last failure: %s", - role.getName(), - role.getFailed(), - role.getStartFailed(), - threshold, - role.getFailureMessage()); + SliderExitCodes.EXIT_DEPLOYMENT_FAILED, FinalApplicationStatus.FAILED, + ErrorStrings.E_UNSTABLE_CLUSTER + + " - failed with component %s failed 'recently' %d times;" + + " threshold is %d - last failure: %s", role.getName(), + role.getFailedRecently(), threshold, role.getFailureMessage()); } } @@ -2000,26 +1486,11 @@ public class AppState { * @return the threshold for failures */ private int getFailureThresholdForRole(RoleStatus roleStatus) { - ConfTreeOperations resources = - instanceDefinition.getResourceOperations(); - return resources.getComponentOptInt(roleStatus.getGroup(), - CONTAINER_FAILURE_THRESHOLD, - failureThreshold); + return (int) roleStatus.getProviderRole().component.getConfiguration() + .getPropertyLong(CONTAINER_FAILURE_THRESHOLD, + DEFAULT_CONTAINER_FAILURE_THRESHOLD); } - /** - * Get the node failure threshold for a specific role, falling back to - * the global one if not - * @param roleGroup role group - * @return the threshold for failures - */ - private int getNodeFailureThresholdForRole(String roleGroup) { - ConfTreeOperations resources = - instanceDefinition.getResourceOperations(); - return resources.getComponentOptInt(roleGroup, - NODE_FAILURE_THRESHOLD, - nodeFailureThreshold); - } /** * Reset the "recent" failure counts of all roles @@ -2027,9 +1498,9 @@ public class AppState { public void resetFailureCounts() { for (RoleStatus roleStatus : getRoleStatusMap().values()) { long failed = roleStatus.resetFailedRecently(); - log.info("Resetting failure count of {}; was {}", - roleStatus.getName(), + log.info("Resetting failure count of {}; was {}", roleStatus.getName(), failed); + } roleHistory.resetFailedRecently(); } @@ -2075,6 +1546,7 @@ public class AppState { @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") private List reviewOneRole(RoleStatus role) throws SliderInternalStateException, TriggerClusterTeardownException { + log.info("review one role " + role.getName()); List operations = new ArrayList<>(); long delta; long expected; @@ -2123,7 +1595,8 @@ public class AppState { log.warn("Awaiting node map before generating anti-affinity requests"); } log.info("Setting pending to {}", pending); - role.setPendingAntiAffineRequests(pending); + //TODO + role.setAAPending((int)pending); } else { for (int i = 0; i < delta; i++) { @@ -2139,7 +1612,7 @@ public class AppState { long excess = -delta; // how many requests are outstanding? for AA roles, this includes pending - long outstandingRequests = role.getRequested() + role.getPendingAntiAffineRequests(); + long outstandingRequests = role.getPending() + role.getAAPending(); if (outstandingRequests > 0) { // outstanding requests. int toCancel = (int)Math.min(outstandingRequests, excess); @@ -2153,8 +1626,7 @@ public class AppState { " expected to be able to cancel {} requests, but got {}", toCancel, cancellations.size()); } - - role.cancel(toCancel); + decPendingContainers(role, toCancel); excess -= toCancel; assert excess >= 0 : "Attempted to cancel too many requests"; log.info("Submitted {} cancellations, leaving {} to release", @@ -2215,9 +1687,9 @@ public class AppState { } else { // actual + requested == desired // there's a special case here: clear all pending AA requests - if (role.getPendingAntiAffineRequests() > 0) { + if (role.getAAPending() > 0) { log.debug("Clearing outstanding pending AA requests"); - role.setPendingAntiAffineRequests(0); + role.setAAPending(0); } } @@ -2268,28 +1740,6 @@ public class AppState { return operations; } - /** - * Find a container running on a specific host -looking - * into the node ID to determine this. - * - * @param node node - * @param roleId role the container must be in - * @return a container or null if there are no containers on this host - * that can be released. - */ - private RoleInstance findRoleInstanceOnHost(NodeInstance node, int roleId) { - Collection targets = cloneOwnedContainerList(); - String hostname = node.hostname; - for (RoleInstance ri : targets) { - if (hostname.equals(RoleHistoryUtils.hostnameOf(ri.container)) - && ri.roleId == roleId - && containersBeingReleased.get(ri.getContainerId()) == null) { - return ri; - } - } - return null; - } - /** * Release all containers. * @return a list of operations to execute @@ -2329,26 +1779,25 @@ public class AppState { * @param assignments the assignments of roles to containers * @param operations any allocation or release operations */ - public synchronized void onContainersAllocated(List allocatedContainers, - List assignments, - List operations) { - assignments.clear(); - operations.clear(); + public synchronized void onContainersAllocated( + List allocatedContainers, + List assignments, + List operations) { List ordered = roleHistory.prepareAllocationList(allocatedContainers); - log.debug("onContainersAllocated(): Total containers allocated = {}", ordered.size()); + log.info("onContainersAllocated(): Total containers allocated = {}", ordered.size()); for (Container container : ordered) { final NodeId nodeId = container.getNodeId(); String containerHostInfo = nodeId.getHost() + ":" + nodeId.getPort(); //get the role final ContainerId cid = container.getId(); final RoleStatus role = lookupRoleStatus(container); - - //dec requested count - role.decRequested(); + decPendingContainers(role); //inc allocated count -this may need to be dropped in a moment, // but us needed to update the logic below - final long allocated = role.incActual(); + MutableGaugeInt containersRunning = role.getComponentMetrics().containersRunning; + final long allocated = containersRunning.value(); + incRunningContainers(role); final long desired = role.getDesired(); final String roleName = role.getName(); @@ -2364,22 +1813,12 @@ public class AppState { log.info("Discarding surplus {} container {} on {}", roleName, cid, containerHostInfo); operations.add(new ContainerReleaseOperation(cid)); //register as a surplus node - surplusNodes.add(cid); - surplusContainers.inc(); - //and, as we aren't binding it to role, dec that role's actual count - role.decActual(); + surplusContainers.add(cid); + role.getComponentMetrics().surplusContainers.incr(); + containersRunning.decr(); } else { - - // Allocation being accepted -so decrement the number of outstanding requests - decOutstandingContainerRequests(); - - log.info("Assigning role {} to container" + - " {}," + - " on {}:{},", - roleName, - cid, - nodeId.getHost(), - nodeId.getPort()); + log.info("Assigning role {} to container" + " {}," + " on {}:{},", + roleName, cid, nodeId.getHost(), nodeId.getPort()); assignments.add(new ContainerAssignment(container, role, outcome)); //add to the history @@ -2392,13 +1831,13 @@ public class AppState { if (node.canHost(role.getKey(), role.getLabelExpression())) { log.error("Assigned node still declares as available {}", node.toFullString() ); } - if (role.getPendingAntiAffineRequests() > 0) { + if (role.getAAPending() > 0) { // still an outstanding AA request: need to issue a new one. log.info("Asking for next container for AA role {}", roleName); if (!addContainerRequest(operations, createAAContainerRequest(role))) { log.info("No capacity in cluster for new requests"); } else { - role.decPendingAntiAffineRequests(); + role.decAAPending(); } log.debug("Current AA role status {}", role); } else { @@ -2437,8 +1876,7 @@ public class AppState { for (Container container : liveContainers) { addRestartedContainer(container); } - clusterStatus.setInfo(StatusKeys.INFO_CONTAINERS_AM_RESTART, - Integer.toString(liveContainers.size())); + app.setNumberOfRunningContainers((long)liveContainers.size()); return true; } @@ -2458,10 +1896,9 @@ public class AppState { // get the role int roleId = ContainerPriority.extractRole(container); - RoleStatus role = - lookupRoleStatus(roleId); + RoleStatus role = lookupRoleStatus(roleId); // increment its count - role.incActual(); + incRunningContainers(role); String roleName = role.getName(); log.info("Rebuilding container {} in role {} on {},", @@ -2495,12 +1932,6 @@ public class AppState { final StringBuilder sb = new StringBuilder("AppState{"); sb.append("applicationLive=").append(applicationLive); sb.append(", live nodes=").append(liveNodes.size()); - sb.append(", startedContainers=").append(startedContainers); - sb.append(", startFailedContainerCount=").append(startFailedContainerCount); - sb.append(", surplusContainers=").append(surplusContainers); - sb.append(", failedContainerCount=").append(failedContainerCount); - sb.append(", outstanding non-AA Container Requests=") - .append(outstandingContainerRequests); sb.append('}'); return sb.toString(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java index a8aa1a2e40b..2dfded89004 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java @@ -24,7 +24,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.api.resource.Application; import org.apache.slider.providers.ProviderRole; import java.util.ArrayList; @@ -38,26 +38,24 @@ import java.util.Map; * are added. */ public class AppStateBindingInfo { - public AggregateConf instanceDefinition; public Configuration serviceConfig = new Configuration(); - public Configuration publishedProviderConf = new Configuration(false); + public Application application = null; public List roles = new ArrayList<>(); public FileSystem fs; public Path historyPath; public List liveContainers = new ArrayList<>(0); - public Map applicationInfo = new HashMap<>(); public ContainerReleaseSelector releaseSelector = new SimpleReleaseSelector(); /** node reports off the RM. */ public List nodeReports = new ArrayList<>(0); public void validate() throws IllegalArgumentException { - Preconditions.checkArgument(instanceDefinition != null, "null instanceDefinition"); Preconditions.checkArgument(serviceConfig != null, "null appmasterConfig"); - Preconditions.checkArgument(publishedProviderConf != null, "null publishedProviderConf"); Preconditions.checkArgument(releaseSelector != null, "null releaseSelector"); Preconditions.checkArgument(roles != null, "null providerRoles"); Preconditions.checkArgument(fs != null, "null fs"); Preconditions.checkArgument(historyPath != null, "null historyDir"); Preconditions.checkArgument(nodeReports != null, "null nodeReports"); + Preconditions.checkArgument(application != null, "null application"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java index 37e9a7fadf7..80464721557 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java @@ -21,14 +21,12 @@ package org.apache.slider.server.appmaster.state; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterNode; +import org.apache.slider.api.resource.Application; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.NodeInformation; import org.apache.slider.api.types.RoleStatistics; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.registry.docstore.PublishedConfigSet; import org.apache.slider.core.registry.docstore.PublishedExportsSet; @@ -130,45 +128,15 @@ public class ProviderAppState implements StateAccessForProviders { } @Override - public ClusterDescription getClusterStatus() { + public Application getApplication() { return appState.getClusterStatus(); } - @Override - public ConfTreeOperations getResourcesSnapshot() { - return appState.getResourcesSnapshot(); - } - - @Override - public ConfTreeOperations getAppConfSnapshot() { - return appState.getAppConfSnapshot(); - } - - @Override - public ConfTreeOperations getInternalsSnapshot() { - return appState.getInternalsSnapshot(); - } - @Override public boolean isApplicationLive() { return appState.isApplicationLive(); } - @Override - public long getSnapshotTime() { - return appState.getSnapshotTime(); - } - - @Override - public AggregateConf getInstanceDefinitionSnapshot() { - return appState.getInstanceDefinitionSnapshot(); - } - - @Override - public AggregateConf getUnresolvedInstanceDefinition() { - return appState.getUnresolvedInstanceDefinition(); - } - @Override public RoleStatus lookupRoleStatus(int key) { return appState.lookupRoleStatus(key); @@ -221,25 +189,15 @@ public class ProviderAppState implements StateAccessForProviders { } @Override - public ClusterDescription refreshClusterStatus() { + public Application refreshClusterStatus() { return appState.refreshClusterStatus(); } - @Override - public List cloneRoleStatusList() { - return appState.cloneRoleStatusList(); - } - @Override public ApplicationLivenessInformation getApplicationLivenessInformation() { return appState.getApplicationLivenessInformation(); } - @Override - public Map getLiveStatistics() { - return appState.getLiveStatistics(); - } - @Override public Map getComponentInfoSnapshot() { return appState.getComponentInfoSnapshot(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java index 38c70f373f3..b6c3675d3ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java @@ -135,17 +135,6 @@ public class RoleHistory { outstandingRequests = new OutstandingRequestTracker(); } - /** - * Register all metrics with the metrics infra - * @param metrics metrics - */ - public void register(MetricsAndMonitoring metrics) { - metrics.register(RoleHistory.class, dirty, "dirty"); - metrics.register(RoleHistory.class, nodesUpdatedTime, "nodes-updated.time"); - metrics.register(RoleHistory.class, nodeUpdateReceived, "nodes-updated.flag"); - metrics.register(RoleHistory.class, thawedDataTime, "thawed.time"); - metrics.register(RoleHistory.class, saveTime, "saved.time"); - } /** * safety check: make sure the role is unique amongst @@ -1102,13 +1091,13 @@ public class RoleHistory { int roleId = role.getKey(); List requests = new ArrayList<>(toCancel); // there may be pending requests which can be cancelled here - long pending = role.getPendingAntiAffineRequests(); + long pending = role.getAAPending(); if (pending > 0) { // there are some pending ones which can be cancelled first long pendingToCancel = Math.min(pending, toCancel); log.info("Cancelling {} pending AA allocations, leaving {}", toCancel, pendingToCancel); - role.setPendingAntiAffineRequests(pending - pendingToCancel); + role.setAAPending(pending - pendingToCancel); toCancel -= pendingToCancel; } if (toCancel > 0 && role.isAARequestOutstanding()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java index 30cfec9e52c..de52f4e3fa5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java @@ -29,6 +29,7 @@ import org.apache.slider.api.ClusterNode; import org.apache.slider.api.proto.Messages; import org.apache.slider.api.types.ContainerInformation; import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.providers.ProviderRole; import java.util.ArrayList; import java.util.Arrays; @@ -40,6 +41,7 @@ import java.util.List; public final class RoleInstance implements Cloneable { public Container container; + public ProviderRole providerRole; /** * Container ID */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java index 98557ce2eaa..8e8546b42d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java @@ -21,19 +21,22 @@ package org.apache.slider.server.appmaster.state; import com.codahale.metrics.Metric; import com.codahale.metrics.MetricSet; import com.google.common.base.Preconditions; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.RoleStatistics; import org.apache.slider.providers.PlacementPolicy; import org.apache.slider.providers.ProviderRole; import org.apache.slider.server.appmaster.management.BoolMetricPredicate; -import org.apache.slider.server.appmaster.management.LongGauge; +import org.apache.slider.server.appmaster.metrics.SliderMetrics; import java.io.Serializable; import java.util.Comparator; import java.util.HashMap; import java.util.Map; +import static org.apache.hadoop.metrics2.lib.Interns.info; + /** * Models the ongoing status of all nodes in an application. * @@ -42,7 +45,7 @@ import java.util.Map; * requires synchronization. Where synchronized access is good is that it allows for * the whole instance to be locked, for updating multiple entries. */ -public final class RoleStatus implements Cloneable, MetricSet { +public final class RoleStatus implements MetricSet { private final String name; private final String group; @@ -53,25 +56,9 @@ public final class RoleStatus implements Cloneable, MetricSet { private final int key; private final ProviderRole providerRole; - private final LongGauge actual = new LongGauge(); - private final LongGauge completed = new LongGauge(); - private final LongGauge desired = new LongGauge(); - private final LongGauge failed = new LongGauge(); - private final LongGauge failedRecently = new LongGauge(0); - private final LongGauge limitsExceeded = new LongGauge(0); - private final LongGauge nodeFailed = new LongGauge(0); - /** Number of AA requests queued. */ - private final LongGauge pendingAntiAffineRequests = new LongGauge(0); - private final LongGauge preempted = new LongGauge(0); - private final LongGauge releasing = new LongGauge(); - private final LongGauge requested = new LongGauge(); - private final LongGauge started = new LongGauge(); - private final LongGauge startFailed = new LongGauge(); - private final LongGauge totalRequested = new LongGauge(); - /** resource requirements */ private Resource resourceRequirements; - + private SliderMetrics componentMetrics; /** any pending AA request */ private volatile OutstandingRequest outstandingAArequest = null; @@ -84,28 +71,19 @@ public final class RoleStatus implements Cloneable, MetricSet { this.name = providerRole.name; this.group = providerRole.group; this.key = providerRole.id; + componentMetrics = + SliderMetrics.register(this.name, "Metrics for component " + this.name); + componentMetrics + .tag("type", "Metrics type [component or service]", "component"); + } + + public SliderMetrics getComponentMetrics() { + return this.componentMetrics; } @Override public Map getMetrics() { Map metrics = new HashMap<>(15); - metrics.put("actual", actual); - metrics.put("completed", completed ); - metrics.put("desired", desired); - metrics.put("failed", failed); - metrics.put("limitsExceeded", limitsExceeded); - metrics.put("nodeFailed", nodeFailed); - metrics.put("preempted", preempted); - metrics.put("pendingAntiAffineRequests", pendingAntiAffineRequests); - metrics.put("releasing", releasing); - metrics.put("requested", requested); - metrics.put("preempted", preempted); - metrics.put("releasing", releasing ); - metrics.put("requested", requested); - metrics.put("started", started); - metrics.put("startFailed", startFailed); - metrics.put("totalRequested", totalRequested); - metrics.put("outstandingAArequest", new BoolMetricPredicate(new BoolMetricPredicate.Eval() { @Override @@ -174,83 +152,6 @@ public final class RoleStatus implements Cloneable, MetricSet { return !hasPlacementPolicy(PlacementPolicy.ANYWHERE); } - public long getDesired() { - return desired.get(); - } - - public void setDesired(long desired) { - this.desired.set(desired); - } - - public long getActual() { - return actual.get(); - } - - public long incActual() { - return actual.incrementAndGet(); - } - - public long decActual() { - return actual.decToFloor(1); - } - - /** - * Get the request count. - * @return a count of requested containers - */ - public long getRequested() { - return requested.get(); - } - - public long incRequested() { - totalRequested.incrementAndGet(); - return requested.incrementAndGet(); - } - - public void cancel(long count) { - requested.decToFloor(count); - } - - public void decRequested() { - cancel(1); - } - - public long getReleasing() { - return releasing.get(); - } - - public long incReleasing() { - return releasing.incrementAndGet(); - } - - public long decReleasing() { - return releasing.decToFloor(1); - } - - public long getFailed() { - return failed.get(); - } - - public long getFailedRecently() { - return failedRecently.get(); - } - - /** - * Reset the recent failure - * @return the number of failures in the "recent" window - */ - public long resetFailedRecently() { - return failedRecently.getAndSet(0); - } - - public long getLimitsExceeded() { - return limitsExceeded.get(); - } - - public long incPendingAntiAffineRequests(long v) { - return pendingAntiAffineRequests.addAndGet(v); - } - /** * Probe for an outstanding AA request being true * @return true if there is an outstanding AA Request @@ -271,94 +172,14 @@ public final class RoleStatus implements Cloneable, MetricSet { * Note that a role failed, text will * be used in any diagnostics if an exception * is later raised. - * @param startupFailure flag to indicate this was a startup event * @param text text about the failure - * @param outcome outcome of the container */ - public synchronized void noteFailed(boolean startupFailure, String text, - ContainerOutcome outcome) { + public synchronized void noteFailed(String text) { if (text != null) { failureMessage = text; } - switch (outcome) { - case Preempted: - preempted.incrementAndGet(); - break; - - case Node_failure: - nodeFailed.incrementAndGet(); - failed.incrementAndGet(); - break; - - case Failed_limits_exceeded: // exceeded memory or CPU; app/configuration related - limitsExceeded.incrementAndGet(); - // fall through - case Failed: // application failure, possibly node related, possibly not - default: // anything else (future-proofing) - failed.incrementAndGet(); - failedRecently.incrementAndGet(); - //have a look to see if it short lived - if (startupFailure) { - incStartFailed(); - } - break; - } } - public long getStartFailed() { - return startFailed.get(); - } - - public synchronized void incStartFailed() { - startFailed.getAndIncrement(); - } - - public synchronized String getFailureMessage() { - return failureMessage; - } - - public long getCompleted() { - return completed.get(); - } - - public long incCompleted() { - return completed.incrementAndGet(); - } - public long getStarted() { - return started.get(); - } - - public synchronized void incStarted() { - started.incrementAndGet(); - } - - public long getTotalRequested() { - return totalRequested.get(); - } - - public long getPreempted() { - return preempted.get(); - } - - public long getNodeFailed() { - return nodeFailed.get(); - } - - public long getPendingAntiAffineRequests() { - return pendingAntiAffineRequests.get(); - } - - public void setPendingAntiAffineRequests(long pendingAntiAffineRequests) { - this.pendingAntiAffineRequests.set(pendingAntiAffineRequests); - } - - public long decPendingAntiAffineRequests() { - return pendingAntiAffineRequests.decToFloor(1); - } - - public OutstandingRequest getOutstandingAArequest() { - return outstandingAArequest; - } public void setOutstandingAArequest(OutstandingRequest outstandingAArequest) { this.outstandingAArequest = outstandingAArequest; @@ -379,11 +200,50 @@ public final class RoleStatus implements Cloneable, MetricSet { public void cancelOutstandingAARequest() { if (outstandingAArequest != null) { setOutstandingAArequest(null); - setPendingAntiAffineRequests(0); - decRequested(); } } + public long getDesired() { + return componentMetrics.containersDesired.value(); + } + + long getRunning() { + return componentMetrics.containersRunning.value(); + } + + public long getPending() { + return componentMetrics.containersPending.value(); + } + + public long getAAPending() { + return componentMetrics.pendingAAContainers.value(); + } + + void decAAPending() { + componentMetrics.pendingAAContainers.decr(); + } + void setAAPending(long n) { + componentMetrics.pendingAAContainers.set((int)n); + } + + long getFailedRecently() { + return componentMetrics.failedSinceLastThreshold.value(); + } + + long resetFailedRecently() { + long count = + componentMetrics.failedSinceLastThreshold.value(); + componentMetrics.failedSinceLastThreshold.set(0); + return count; + } + + long getFailed() { + return componentMetrics.containersFailed.value(); + } + + String getFailureMessage() { + return this.failureMessage; + } /** * Get the number of roles we are short of. * nodes released are ignored. @@ -392,10 +252,9 @@ public final class RoleStatus implements Cloneable, MetricSet { */ public long getDelta() { long inuse = getActualAndRequested(); - long delta = desired.get() - inuse; + long delta = getDesired() - inuse; if (delta < 0) { //if we are releasing, remove the number that are already released. - delta += releasing.get(); //but never switch to a positive delta = Math.min(delta, 0); } @@ -407,43 +266,7 @@ public final class RoleStatus implements Cloneable, MetricSet { * @return the size of the application when outstanding requests are included. */ public long getActualAndRequested() { - return actual.get() + requested.get(); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("RoleStatus{"); - sb.append("name='").append(name).append('\''); - sb.append(", group=").append(group); - sb.append(", key=").append(key); - sb.append(", desired=").append(desired); - sb.append(", actual=").append(actual); - sb.append(", requested=").append(requested); - sb.append(", releasing=").append(releasing); - sb.append(", failed=").append(failed); - sb.append(", startFailed=").append(startFailed); - sb.append(", started=").append(started); - sb.append(", completed=").append(completed); - sb.append(", totalRequested=").append(totalRequested); - sb.append(", preempted=").append(preempted); - sb.append(", nodeFailed=").append(nodeFailed); - sb.append(", failedRecently=").append(failedRecently); - sb.append(", limitsExceeded=").append(limitsExceeded); - sb.append(", resourceRequirements=").append(resourceRequirements); - sb.append(", isAntiAffinePlacement=").append(isAntiAffinePlacement()); - if (isAntiAffinePlacement()) { - sb.append(", pendingAntiAffineRequests=").append(pendingAntiAffineRequests); - sb.append(", outstandingAArequest=").append(outstandingAArequest); - } - sb.append(", failureMessage='").append(failureMessage).append('\''); - sb.append(", providerRole=").append(providerRole); - sb.append('}'); - return sb.toString(); - } - - @Override - public synchronized Object clone() throws CloneNotSupportedException { - return super.clone(); + return getRunning() + getPending(); } /** @@ -454,15 +277,6 @@ public final class RoleStatus implements Cloneable, MetricSet { return providerRole; } - /** - * Build the statistics map from the current data - * @return a map for use in statistics reports - */ - public Map buildStatistics() { - ComponentInformation componentInformation = serialize(); - return componentInformation.buildStatistics(); - } - /** * Produced a serialized form which can be served up as JSON * @return a summary of the current role status. @@ -470,21 +284,6 @@ public final class RoleStatus implements Cloneable, MetricSet { public synchronized ComponentInformation serialize() { ComponentInformation info = new ComponentInformation(); info.name = name; - info.priority = getPriority(); - info.desired = desired.intValue(); - info.actual = actual.intValue(); - info.requested = requested.intValue(); - info.releasing = releasing.intValue(); - info.failed = failed.intValue(); - info.startFailed = startFailed.intValue(); - info.placementPolicy = getPlacementPolicy(); - info.failureMessage = failureMessage; - info.totalRequested = totalRequested.intValue(); - info.failedRecently = failedRecently.intValue(); - info.nodeFailed = nodeFailed.intValue(); - info.preempted = preempted.intValue(); - info.pendingAntiAffineRequestCount = pendingAntiAffineRequests.intValue(); - info.isAARequestOutstanding = isAARequestOutstanding(); return info; } @@ -542,17 +341,6 @@ public final class RoleStatus implements Cloneable, MetricSet { public synchronized RoleStatistics getStatistics() { RoleStatistics stats = new RoleStatistics(); stats.activeAA = getOutstandingAARequestCount(); - stats.actual = actual.get(); - stats.desired = desired.get(); - stats.failed = failed.get(); - stats.limitsExceeded = limitsExceeded.get(); - stats.nodeFailed = nodeFailed.get(); - stats.preempted = preempted.get(); - stats.releasing = releasing.get(); - stats.requested = requested.get(); - stats.started = started.get(); - stats.startFailed = startFailed.get(); - stats.totalRequested = totalRequested.get(); return stats; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java index ad91183a766..118ca9d6fdb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java @@ -21,15 +21,13 @@ package org.apache.slider.server.appmaster.state; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterNode; import org.apache.slider.api.StatusKeys; +import org.apache.slider.api.resource.Application; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.NodeInformation; import org.apache.slider.api.types.RoleStatistics; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTreeOperations; import org.apache.slider.core.exceptions.NoSuchNodeException; import org.apache.slider.core.registry.docstore.PublishedConfigSet; import org.apache.slider.core.registry.docstore.PublishedExportsSet; @@ -105,29 +103,7 @@ public interface StateAccessForProviders { * Get the current cluster description * @return the actual state of the cluster */ - ClusterDescription getClusterStatus(); - - /** - * Get at the snapshot of the resource config - * Changes here do not affect the application state. - * @return the most recent settings - */ - ConfTreeOperations getResourcesSnapshot(); - - /** - * Get at the snapshot of the appconf config - * Changes here do not affect the application state. - * @return the most recent settings - */ - ConfTreeOperations getAppConfSnapshot(); - - /** - * Get at the snapshot of the internals config. - * Changes here do not affect the application state. - * @return the internals settings - */ - - ConfTreeOperations getInternalsSnapshot(); + Application getApplication(); /** * Flag set to indicate the application is live -this only happens @@ -135,22 +111,8 @@ public interface StateAccessForProviders { */ boolean isApplicationLive(); - long getSnapshotTime(); - /** - * Get a snapshot of the entire aggregate configuration - * @return the aggregate configuration - */ - AggregateConf getInstanceDefinitionSnapshot(); - - /** - * Get the desired/unresolved value - * @return unresolved - */ - AggregateConf getUnresolvedInstanceDefinition(); - - /** - * Look up a role from its key -or fail + * Look up a role from its key -or fail * * @param key key to resolve * @return the status @@ -159,7 +121,7 @@ public interface StateAccessForProviders { RoleStatus lookupRoleStatus(int key); /** - * Look up a role from its key -or fail + * Look up a role from its key -or fail * * @param c container in a role * @return the status @@ -232,14 +194,8 @@ public interface StateAccessForProviders { /** * Update the cluster description with anything interesting */ - ClusterDescription refreshClusterStatus(); + Application refreshClusterStatus(); - /** - * Get a deep clone of the role status list. Concurrent events may mean this - * list (or indeed, some of the role status entries) may be inconsistent - * @return a snapshot of the role status entries - */ - List cloneRoleStatusList(); /** * get application liveness information @@ -247,13 +203,6 @@ public interface StateAccessForProviders { */ ApplicationLivenessInformation getApplicationLivenessInformation(); - /** - * Get the live statistics map - * @return a map of statistics values, defined in the {@link StatusKeys} - * keylist. - */ - Map getLiveStatistics(); - /** * Get a snapshot of component information. *

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java index 7ecc00c91a4..0cac430504f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java @@ -16,6 +16,8 @@ */ package org.apache.slider.server.appmaster.web; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.health.HealthCheckRegistry; import com.codahale.metrics.servlets.HealthCheckServlet; import com.codahale.metrics.servlets.MetricsServlet; import com.codahale.metrics.servlets.PingServlet; @@ -28,10 +30,8 @@ import com.sun.jersey.spi.container.servlet.ServletContainer; import org.apache.hadoop.yarn.webapp.Dispatcher; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; -import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.web.rest.AMWadlGeneratorConfig; import org.apache.slider.server.appmaster.web.rest.AMWebServices; -import static org.apache.slider.server.appmaster.web.rest.RestPaths.*; import org.apache.slider.server.appmaster.web.rest.SliderJacksonJaxbJsonProvider; import java.util.HashMap; @@ -39,6 +39,8 @@ import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.*; + /** * */ @@ -65,6 +67,7 @@ public class SliderAMWebApp extends WebApp { bind(GenericExceptionHandler.class); // bind the REST interface bind(AMWebServices.class); + //bind(AMAgentWebServices.class); route("/", SliderAMController.class); route(CONTAINER_STATS, SliderAMController.class, "containerStats"); @@ -81,11 +84,9 @@ public class SliderAMWebApp extends WebApp { serve(path).with(Dispatcher.class); } - // metrics - MetricsAndMonitoring monitoring = - webAppApi.getMetricsAndMonitoring(); - serve(SYSTEM_HEALTHCHECK).with(new HealthCheckServlet(monitoring.getHealth())); - serve(SYSTEM_METRICS).with(new MetricsServlet(monitoring.getMetrics())); + serve(SYSTEM_HEALTHCHECK) + .with(new HealthCheckServlet(new HealthCheckRegistry())); + serve(SYSTEM_METRICS).with(new MetricsServlet(new MetricRegistry())); serve(SYSTEM_PING).with(new PingServlet()); serve(SYSTEM_THREADS).with(new ThreadDumpServlet()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java index ea07a8aaa0a..094726d1c42 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java @@ -22,7 +22,6 @@ import org.apache.slider.server.appmaster.actions.QueueAccess; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.state.AppState; import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache; /** * Interface to pass information from the Slider AppMaster to the WebApp @@ -56,10 +55,4 @@ public interface WebAppApi { * @return the immediate and scheduled queues */ QueueAccess getQueues(); - - /** - * Local cache of content - * @return the cache - */ - ContentCache getContentCache(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java index d20f1adb82b..fd9381c5c96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java @@ -18,11 +18,9 @@ package org.apache.slider.server.appmaster.web; import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.slider.providers.ProviderService; -import org.apache.slider.server.appmaster.AppMasterActionOperations; import org.apache.slider.server.appmaster.actions.QueueAccess; import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,15 +37,10 @@ public class WebAppApiImpl implements WebAppApi { private final RegistryOperations registryOperations; private final MetricsAndMonitoring metricsAndMonitoring; private final QueueAccess queues; - private final AppMasterActionOperations appMasterOperations; - private final ContentCache contentCache; public WebAppApiImpl(StateAccessForProviders appState, ProviderService provider, RegistryOperations registryOperations, - MetricsAndMonitoring metricsAndMonitoring, QueueAccess queues, - AppMasterActionOperations appMasterOperations, ContentCache contentCache) { - this.appMasterOperations = appMasterOperations; - this.contentCache = contentCache; + MetricsAndMonitoring metricsAndMonitoring, QueueAccess queues) { checkNotNull(appState); checkNotNull(provider); this.queues = queues; @@ -82,10 +75,4 @@ public class WebAppApiImpl implements WebAppApi { public QueueAccess getQueues() { return queues; } - - - @Override - public ContentCache getContentCache() { - return contentCache; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java index aed87d8455e..e73dd875204 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java @@ -18,13 +18,24 @@ package org.apache.slider.server.appmaster.web.rest; import com.google.inject.Inject; import com.google.inject.Singleton; +import org.apache.slider.api.resource.Application; import org.apache.slider.server.appmaster.web.WebAppApi; -import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource; +import org.apache.slider.server.appmaster.web.rest.application.actions.RestActionStop; +import org.apache.slider.server.appmaster.web.rest.application.actions.StopResponse; import org.apache.slider.server.appmaster.web.rest.management.ManagementResource; import org.apache.slider.server.appmaster.web.rest.publisher.PublisherResource; import org.apache.slider.server.appmaster.web.rest.registry.RegistryResource; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.GET; +import javax.ws.rs.POST; import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.UriInfo; + +import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import static org.apache.slider.server.appmaster.web.rest.RestPaths.ACTION_STOP; /** * The available REST services exposed by a slider AM. @@ -38,7 +49,6 @@ public class AMWebServices { private final ManagementResource managementResource; private final PublisherResource publisherResource; private final RegistryResource registryResource; - private final ApplicationResource applicationResource; @Inject public AMWebServices(WebAppApi slider) { @@ -46,7 +56,6 @@ public class AMWebServices { managementResource = new ManagementResource(slider); publisherResource = new PublisherResource(slider); registryResource = new RegistryResource(slider); - applicationResource = new ApplicationResource(slider); } @Path(RestPaths.SLIDER_SUBPATH_MANAGEMENT) @@ -63,9 +72,21 @@ public class AMWebServices { public RegistryResource getRegistryResource() { return registryResource; } - + + + @GET @Path(RestPaths.SLIDER_SUBPATH_APPLICATION) - public ApplicationResource getApplicationResource() { - return applicationResource; + @Produces({APPLICATION_JSON}) + public Application getApplicationResource() { + return slider.getAppState().getApplication(); + } + + @POST + @Path(ACTION_STOP) + @Produces({APPLICATION_JSON}) + public StopResponse actionStop(@Context HttpServletRequest request, + @Context UriInfo uriInfo, + String body) { + return new RestActionStop(slider).stop(request, uriInfo, body); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java index ae9eb0f0343..581f5b429ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java @@ -34,32 +34,14 @@ public class RestPaths { /** * agent content root: {@value} */ - public static final String WS_AGENT_CONTEXT_ROOT = "/" + AGENT_WS_CONTEXT; - public static final String V1_SLIDER = "/v1/slider"; - public static final String SLIDER_CONTEXT_ROOT = WS_CONTEXT_ROOT + V1_SLIDER; - public static final String RELATIVE_API = WS_CONTEXT + V1_SLIDER; - public static final String SLIDER_AGENT_CONTEXT_ROOT = WS_AGENT_CONTEXT_ROOT + V1_SLIDER; + public static final String SLIDER_CONTEXT_ROOT = WS_CONTEXT_ROOT + "/v1"; + public static final String RELATIVE_API = WS_CONTEXT + "/v1"; public static final String MANAGEMENT = "mgmt"; public static final String SLIDER_SUBPATH_MANAGEMENT = "/" + MANAGEMENT; public static final String SLIDER_SUBPATH_AGENTS = "/agents"; public static final String SLIDER_SUBPATH_PUBLISHER = "/publisher"; - /** - * management path: {@value} - */ - public static final String SLIDER_PATH_MANAGEMENT = SLIDER_CONTEXT_ROOT - + SLIDER_SUBPATH_MANAGEMENT; - - public static final String RELATIVE_PATH_MANAGEMENT = RELATIVE_API - + SLIDER_SUBPATH_MANAGEMENT; - - /** - * Agents: {@value} - */ - public static final String SLIDER_PATH_AGENTS = SLIDER_AGENT_CONTEXT_ROOT - + SLIDER_SUBPATH_AGENTS; - /** * Publisher: {@value} */ @@ -105,6 +87,7 @@ public class RestPaths { public static final String SYSTEM = "/system"; + /** * Codahale Metrics - health: {@value} */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java index 2facf16db78..d23fceeee0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java @@ -19,17 +19,7 @@ package org.apache.slider.server.appmaster.web.rest.application; import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.appmaster.web.rest.application.resources.AggregateModelRefresher; -import org.apache.slider.server.appmaster.web.rest.application.resources.AppconfRefresher; -import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent; import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache; -import org.apache.slider.server.appmaster.web.rest.application.resources.LiveComponentsRefresher; -import org.apache.slider.server.appmaster.web.rest.application.resources.LiveContainersRefresher; -import org.apache.slider.server.appmaster.web.rest.application.resources.LiveNodesRefresher; -import org.apache.slider.server.appmaster.web.rest.application.resources.LiveResourcesRefresher; -import org.apache.slider.server.appmaster.web.rest.application.resources.LiveStatisticsRefresher; - -import static org.apache.slider.server.appmaster.web.rest.RestPaths.*; public class ApplicationResouceContentCacheFactory { public static final int LIFESPAN = 500; @@ -41,23 +31,6 @@ public class ApplicationResouceContentCacheFactory { public static ContentCache createContentCache( StateAccessForProviders state) { ContentCache cache = new ContentCache(); - cache.put(LIVE_RESOURCES, new CachedContent<>(LIFESPAN, new LiveResourcesRefresher(state))); - cache.put(LIVE_CONTAINERS, new CachedContent<>(LIFESPAN, new LiveContainersRefresher(state))); - cache.put(LIVE_COMPONENTS, new CachedContent<>(LIFESPAN, new LiveComponentsRefresher(state))); - cache.put(LIVE_NODES, new CachedContent<>(LIFESPAN, new LiveNodesRefresher(state))); - cache.put(MODEL_DESIRED, - new CachedContent<>(LIFESPAN, new AggregateModelRefresher(state, false))); - cache.put(MODEL_RESOLVED, - new CachedContent<>(LIFESPAN, new AggregateModelRefresher(state, true))); - cache.put(MODEL_RESOLVED_APPCONF, - new CachedContent<>(LIFESPAN, new AppconfRefresher(state, false, false))); - cache.put(MODEL_RESOLVED_RESOURCES, - new CachedContent<>(LIFESPAN, new AppconfRefresher(state, false, true))); - cache.put(MODEL_DESIRED_APPCONF, - new CachedContent<>(LIFESPAN, new AppconfRefresher(state, true, false))); - cache.put(MODEL_DESIRED_RESOURCES, - new CachedContent<>(LIFESPAN, new AppconfRefresher(state, true, true))); - cache.put(LIVE_STATISTICS, new CachedContent<>(LIFESPAN, new LiveStatisticsRefresher(state))); return cache; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java deleted file mode 100644 index 52068d6b19a..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java +++ /dev/null @@ -1,516 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.web.rest.application; - -import com.google.common.collect.Lists; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.webapp.BadRequestException; -import org.apache.hadoop.yarn.webapp.NotFoundException; -import org.apache.slider.api.types.ApplicationLivenessInformation; -import org.apache.slider.api.types.ComponentInformation; -import org.apache.slider.api.types.ContainerInformation; -import org.apache.slider.api.types.NodeInformation; -import org.apache.slider.api.types.NodeInformationList; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTree; -import org.apache.slider.core.exceptions.NoSuchNodeException; -import org.apache.slider.core.persist.ConfTreeSerDeser; -import org.apache.slider.server.appmaster.actions.ActionFlexCluster; -import org.apache.slider.server.appmaster.actions.AsyncAction; -import org.apache.slider.server.appmaster.actions.QueueAccess; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.appmaster.web.WebAppApi; -import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource; -import static org.apache.slider.server.appmaster.web.rest.RestPaths.*; - -import org.apache.slider.server.appmaster.web.rest.application.actions.RestActionStop; -import org.apache.slider.server.appmaster.web.rest.application.actions.StopResponse; -import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache; -import org.apache.slider.server.appmaster.web.rest.application.actions.RestActionPing; -import org.apache.slider.api.types.PingInformation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Singleton; -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.Consumes; -import javax.ws.rs.DELETE; -import javax.ws.rs.GET; -import javax.ws.rs.HEAD; -import javax.ws.rs.POST; -import javax.ws.rs.PUT; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.Context; - -import static javax.ws.rs.core.MediaType.*; - -import javax.ws.rs.core.Response; -import javax.ws.rs.core.UriInfo; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -@Singleton -@SuppressWarnings("unchecked") -public class ApplicationResource extends AbstractSliderResource { - private static final Logger log = - LoggerFactory.getLogger(ApplicationResource.class); - - public static final List LIVE_ENTRIES = toJsonList("resources", - "containers", - "components", - "nodes", - "statistics", - "internal"); - - public static final List ROOT_ENTRIES = - toJsonList("model", "live", "actions"); - - public static final List MODEL_ENTRIES = - toJsonList("desired", "resolved"); - - /** - * This is the cache of all content ... each entry is - * designed to be self-refreshing on get operations, - * so is never very out of date, yet many GETs don't - * overload the rest of the system. - */ - private final ContentCache cache; - private final StateAccessForProviders state; - private final QueueAccess actionQueues; - - public ApplicationResource(WebAppApi slider) { - super(slider); - state = slider.getAppState(); - cache = slider.getContentCache(); - actionQueues = slider.getQueues(); - } - - /** - * Build a new JSON-marshallable list of string elements - * @param elements elements - * @return something that can be returned - */ - private static List toJsonList(String... elements) { - return Lists.newArrayList(elements); - } - - @GET - @Path("/") - @Produces({APPLICATION_JSON}) - public List getRoot() { - markGet(SLIDER_SUBPATH_APPLICATION); - return ROOT_ENTRIES; - } - - /** - * Enum model values: desired and resolved - * @return the desired and resolved model - */ - @GET - @Path(MODEL) - @Produces({APPLICATION_JSON}) - public List getModel() { - markGet(SLIDER_SUBPATH_APPLICATION, MODEL); - return MODEL_ENTRIES; - } - - @GET - @Path(MODEL_DESIRED) - @Produces({APPLICATION_JSON}) - public AggregateConf getModelDesired() { - markGet(SLIDER_SUBPATH_APPLICATION, MODEL_DESIRED); - return lookupAggregateConf(MODEL_DESIRED); - } - - @GET - @Path(MODEL_DESIRED_APPCONF) - @Produces({APPLICATION_JSON}) - public ConfTree getModelDesiredAppconf() { - markGet(SLIDER_SUBPATH_APPLICATION, MODEL_DESIRED_APPCONF); - return lookupConfTree(MODEL_DESIRED_APPCONF); - } - - @GET - @Path(MODEL_DESIRED_RESOURCES) - @Produces({APPLICATION_JSON}) - public ConfTree getModelDesiredResources() { - markGet(SLIDER_SUBPATH_APPLICATION, MODEL_DESIRED_RESOURCES); - return lookupConfTree(MODEL_DESIRED_RESOURCES); - } - -/* - @PUT - @Path(MODEL_DESIRED_RESOURCES) -// @Consumes({APPLICATION_JSON, TEXT_PLAIN}) - @Consumes({TEXT_PLAIN}) - @Produces({APPLICATION_JSON}) -*/ - public ConfTree setModelDesiredResources( - String json) { - markPut(SLIDER_SUBPATH_APPLICATION, MODEL_DESIRED_RESOURCES); - int size = json != null ? json.length() : 0; - log.info("PUT {} {} bytes:\n{}", MODEL_DESIRED_RESOURCES, - size, - json); - if (size == 0) { - log.warn("No JSON in PUT request; rejecting"); - throw new BadRequestException("No JSON in PUT"); - } - - try { - ConfTreeSerDeser serDeser = new ConfTreeSerDeser(); - ConfTree updated = serDeser.fromJson(json); - queue(new ActionFlexCluster("flex", - 1, TimeUnit.MILLISECONDS, - updated)); - // return the updated value, even though it potentially hasn't yet - // been executed - return updated; - } catch (Exception e) { - throw buildException("PUT to "+ MODEL_DESIRED_RESOURCES , e); - } - } - @PUT - @Path(MODEL_DESIRED_RESOURCES) - @Consumes({APPLICATION_JSON}) - @Produces({APPLICATION_JSON}) - public ConfTree setModelDesiredResources( - ConfTree updated) { - try { - queue(new ActionFlexCluster("flex", - 1, TimeUnit.MILLISECONDS, - updated)); - // return the updated value, even though it potentially hasn't yet - // been executed - return updated; - } catch (Exception e) { - throw buildException("PUT to "+ MODEL_DESIRED_RESOURCES , e); - } - } - - - - @GET - @Path(MODEL_RESOLVED) - @Produces({APPLICATION_JSON}) - public AggregateConf getModelResolved() { - markGet(SLIDER_SUBPATH_APPLICATION, MODEL_RESOLVED); - return lookupAggregateConf(MODEL_RESOLVED); - } - - @GET - @Path(MODEL_RESOLVED_APPCONF) - @Produces({APPLICATION_JSON}) - public ConfTree getModelResolvedAppconf() { - markGet(SLIDER_SUBPATH_APPLICATION, MODEL_RESOLVED_APPCONF); - return lookupConfTree(MODEL_RESOLVED_APPCONF); - } - - @GET - @Path(MODEL_RESOLVED_RESOURCES) - @Produces({APPLICATION_JSON}) - public ConfTree getModelResolvedResources() { - markGet(SLIDER_SUBPATH_APPLICATION, MODEL_RESOLVED_RESOURCES); - return lookupConfTree(MODEL_RESOLVED_RESOURCES); - } - - @GET - @Path(LIVE) - @Produces({APPLICATION_JSON}) - public List getLive() { - markGet(SLIDER_SUBPATH_APPLICATION, LIVE); - return LIVE_ENTRIES; - } - - @GET - @Path(LIVE_RESOURCES) - @Produces({APPLICATION_JSON}) - public ConfTree getLiveResources() { - markGet(SLIDER_SUBPATH_APPLICATION, LIVE_RESOURCES); - return lookupConfTree(LIVE_RESOURCES); - } - - @GET - @Path(LIVE_CONTAINERS) - @Produces({APPLICATION_JSON}) - public Map getLiveContainers() { - markGet(SLIDER_SUBPATH_APPLICATION, LIVE_CONTAINERS); - try { - return (Map)cache.lookup( - LIVE_CONTAINERS); - } catch (Exception e) { - throw buildException(LIVE_CONTAINERS, e); - } - } - - @GET - @Path(LIVE_CONTAINERS + "/{containerId}") - @Produces({APPLICATION_JSON}) - public ContainerInformation getLiveContainer( - @PathParam("containerId") String containerId) { - markGet(SLIDER_SUBPATH_APPLICATION, LIVE_CONTAINERS); - try { - RoleInstance id = state.getLiveInstanceByContainerID(containerId); - return id.serialize(); - } catch (NoSuchNodeException e) { - throw new NotFoundException("Unknown container: " + containerId); - } catch (Exception e) { - throw buildException(LIVE_CONTAINERS + "/"+ containerId, e); - } - } - - @GET - @Path(LIVE_COMPONENTS) - @Produces({APPLICATION_JSON}) - public Map getLiveComponents() { - markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS); - try { - return (Map) cache.lookup(LIVE_COMPONENTS); - } catch (Exception e) { - throw buildException(LIVE_COMPONENTS, e); - } - } - - @GET - @Path(LIVE_COMPONENTS + "/{component}") - @Produces({APPLICATION_JSON}) - public ComponentInformation getLiveComponent( - @PathParam("component") String component) { - markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS); - try { - return state.getComponentInformation(component); - } catch (YarnRuntimeException e) { - throw new NotFoundException("Unknown component: " + component); - } catch (Exception e) { - throw buildException(LIVE_CONTAINERS +"/" + component, e); - } - } - - /** - * Liveness information for the application as a whole - * @return snapshot of liveness - */ - @GET - @Path(LIVE_LIVENESS) - @Produces({APPLICATION_JSON}) - public ApplicationLivenessInformation getLivenessInformation() { - markGet(SLIDER_SUBPATH_APPLICATION, LIVE_LIVENESS); - try { - return state.getApplicationLivenessInformation(); - } catch (Exception e) { - throw buildException(LIVE_CONTAINERS, e); - } - } - -/* -TODO: decide what structure to return here, then implement - - @GET - @Path(LIVE_LIVENESS + "/{component}") - @Produces({APPLICATION_JSON}) - public ApplicationLivenessInformation getLivenessForComponent( - @PathParam("component") String component) { - markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS); - try { - RoleStatus roleStatus = state.lookupRoleStatus(component); - ApplicationLivenessInformation info = new ApplicationLivenessInformation(); - info.requested = roleStatus.getRequested(); - info.allRequestsSatisfied = info.requested == 0; - return info; - } catch (YarnRuntimeException e) { - throw new NotFoundException("Unknown component: " + component); - } catch (Exception e) { - throw buildException(LIVE_LIVENESS + "/" + component, e); - } - } -*/ - - - @GET - @Path(LIVE_NODES) - @Produces({APPLICATION_JSON}) - public NodeInformationList getLiveNodes() { - markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS); - try { - return (NodeInformationList) cache.lookup(LIVE_NODES); - } catch (Exception e) { - throw buildException(LIVE_COMPONENTS, e); - } - } - - @GET - @Path(LIVE_NODES + "/{hostname}") - @Produces({APPLICATION_JSON}) - public NodeInformation getLiveNode(@PathParam("hostname") String hostname) { - markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS); - try { - NodeInformation ni = state.getNodeInformation(hostname); - if (ni != null) { - return ni; - } else { - throw new NotFoundException("Unknown node: " + hostname); - } - } catch (NotFoundException e) { - throw e; - } catch (Exception e) { - throw buildException(LIVE_COMPONENTS + "/" + hostname, e); - } - } - - /** - * Statistics of the application - * @return snapshot statistics - */ - @GET - @Path(LIVE_STATISTICS) - @Produces({APPLICATION_JSON}) - public Map getLiveStatistics() { - markGet(SLIDER_SUBPATH_APPLICATION, LIVE_LIVENESS); - try { - return (Map) cache.lookup(LIVE_STATISTICS); - } catch (Exception e) { - throw buildException(LIVE_STATISTICS, e); - } - } - - /** - * Helper method; look up an aggregate configuration in the cache from - * a key, or raise an exception - * @param key key to resolve - * @return the configuration - * @throws WebApplicationException on a failure - */ - protected AggregateConf lookupAggregateConf(String key) { - try { - return (AggregateConf) cache.lookup(key); - } catch (Exception e) { - throw buildException(key, e); - } - } - - - /** - * Helper method; look up an conf tree in the cache from - * a key, or raise an exception - * @param key key to resolve - * @return the configuration - * @throws WebApplicationException on a failure - */ - protected ConfTree lookupConfTree(String key) { - try { - return (ConfTree) cache.lookup(key); - } catch (Exception e) { - throw buildException(key, e); - } - } - - /* ************************************************************************ - - ACTION PING - - **************************************************************************/ - - @GET - @Path(ACTION_PING) - @Produces({APPLICATION_JSON}) - public PingInformation actionPingGet(@Context HttpServletRequest request, - @Context UriInfo uriInfo) { - markGet(SLIDER_SUBPATH_APPLICATION, ACTION_PING); - return new RestActionPing().ping(request, uriInfo, ""); - } - - @POST - @Path(ACTION_PING) - @Produces({APPLICATION_JSON}) - public PingInformation actionPingPost(@Context HttpServletRequest request, - @Context UriInfo uriInfo, - String body) { - markPost(SLIDER_SUBPATH_APPLICATION, ACTION_PING); - return new RestActionPing().ping(request, uriInfo, body); - } - - @PUT - @Path(ACTION_PING) - @Consumes({TEXT_PLAIN}) - @Produces({APPLICATION_JSON}) - public PingInformation actionPingPut(@Context HttpServletRequest request, - @Context UriInfo uriInfo, - String body) { - markPut(SLIDER_SUBPATH_APPLICATION, ACTION_PING); - return new RestActionPing().ping(request, uriInfo, body); - } - - @DELETE - @Path(ACTION_PING) - @Consumes({APPLICATION_JSON}) - @Produces({APPLICATION_JSON}) - public PingInformation actionPingDelete(@Context HttpServletRequest request, - @Context UriInfo uriInfo) { - markDelete(SLIDER_SUBPATH_APPLICATION, ACTION_PING); - return new RestActionPing().ping(request, uriInfo, ""); - } - - @HEAD - @Path(ACTION_PING) - public Object actionPingHead(@Context HttpServletRequest request, - @Context UriInfo uriInfo) { - mark("HEAD", SLIDER_SUBPATH_APPLICATION, ACTION_PING); - return new RestActionPing().ping(request, uriInfo, ""); - } - - /* ************************************************************************ - - ACTION STOP - - **************************************************************************/ - - - @POST - @Path(ACTION_STOP) - @Produces({APPLICATION_JSON}) - public StopResponse actionStop(@Context HttpServletRequest request, - @Context UriInfo uriInfo, - String body) { - markPost(SLIDER_SUBPATH_APPLICATION, ACTION_STOP); - return new RestActionStop(slider).stop(request, uriInfo, body); - } - - /** - * Schedule an action - * @param action for delayed execution - */ - public void schedule(AsyncAction action) { - actionQueues.schedule(action); - } - - /** - * Put an action on the immediate queue -to be executed when the queue - * reaches it. - * @param action action to queue - */ - public void queue(AsyncAction action) { - actionQueues.put(action); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AggregateModelRefresher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AggregateModelRefresher.java index ee28abf216e..261e66e29be 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AggregateModelRefresher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AggregateModelRefresher.java @@ -23,7 +23,6 @@ import org.apache.slider.server.appmaster.state.StateAccessForProviders; /** * Refresh the aggregate desired model via - * {@link StateAccessForProviders#getInstanceDefinitionSnapshot()} */ public class AggregateModelRefresher implements ResourceRefresher { @@ -39,9 +38,6 @@ public class AggregateModelRefresher @Override public AggregateConf refresh() throws Exception { - return - resolved ? - state.getInstanceDefinitionSnapshot() - : state.getUnresolvedInstanceDefinition(); + return new AggregateConf(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AppconfRefresher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AppconfRefresher.java index 06460ccf52f..190a51e995c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AppconfRefresher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/AppconfRefresher.java @@ -44,10 +44,7 @@ public class AppconfRefresher @Override public ConfTree refresh() throws Exception { - AggregateConf aggregateConf = - unresolved ? - state.getUnresolvedInstanceDefinition(): - state.getInstanceDefinitionSnapshot(); + AggregateConf aggregateConf = new AggregateConf(); ConfTree ct = resources ? aggregateConf.getResources() : aggregateConf.getAppConf(); return new ConfTreeSerDeser().fromInstance(ct); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java deleted file mode 100644 index f9882972c31..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveResourcesRefresher.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.web.rest.application.resources; - -import org.apache.slider.api.StatusKeys; -import org.apache.slider.core.conf.ConfTree; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.server.appmaster.state.RoleStatus; -import org.apache.slider.server.appmaster.state.StateAccessForProviders; - -import java.util.Map; - -public class LiveResourcesRefresher implements ResourceRefresher { - - private final StateAccessForProviders state; - - public LiveResourcesRefresher(StateAccessForProviders state) { - this.state = state; - } - - @Override - public ConfTree refresh() throws Exception { - - // snapshot resources - ConfTreeOperations resources = state.getResourcesSnapshot(); - // then add actual values - Map roleStatusMap = state.getRoleStatusMap(); - - for (RoleStatus status : roleStatusMap.values()) { - String name = status.getName(); - resources.setComponentOpt(name, - StatusKeys.COMPONENT_INSTANCES_REQUESTING, - status.getRequested()); - resources.setComponentOpt(name, - StatusKeys.COMPONENT_INSTANCES_ACTUAL, - status.getActual()); - resources.setComponentOpt(name, - StatusKeys.COMPONENT_INSTANCES_RELEASING, - status.getReleasing()); - resources.setComponentOpt(name, - StatusKeys.COMPONENT_INSTANCES_FAILED, - status.getFailed()); - resources.setComponentOpt(name, - StatusKeys.COMPONENT_INSTANCES_COMPLETED, - status.getCompleted()); - resources.setComponentOpt(name, - StatusKeys.COMPONENT_INSTANCES_STARTED, - status.getStarted()); - } - return resources.getConfTree(); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveStatisticsRefresher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveStatisticsRefresher.java deleted file mode 100644 index d31b455847a..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveStatisticsRefresher.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.web.rest.application.resources; - -import org.apache.slider.server.appmaster.state.StateAccessForProviders; - -import java.util.Map; - -public class LiveStatisticsRefresher implements ResourceRefresher> { - - private final StateAccessForProviders state; - - public LiveStatisticsRefresher(StateAccessForProviders state) { - this.state = state; - } - - @Override - public Map refresh() throws Exception { - - // snapshot resources - return state.getLiveStatistics(); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceSnapshotRefresher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceSnapshotRefresher.java deleted file mode 100644 index c16912afff1..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ResourceSnapshotRefresher.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.web.rest.application.resources; - -import org.apache.slider.core.conf.ConfTree; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.server.appmaster.state.StateAccessForProviders; - -public class ResourceSnapshotRefresher implements ResourceRefresher { - - private final StateAccessForProviders state; - - public ResourceSnapshotRefresher(StateAccessForProviders state) { - this.state = state; - } - - @Override - public ConfTree refresh() throws Exception { - - // snapshot resources - ConfTreeOperations resources = state.getResourcesSnapshot(); - return resources.getConfTree(); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java index f27711a2829..14d94007c93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/ManagementResource.java @@ -88,6 +88,7 @@ public class ManagementResource extends AbstractSliderResource { } protected AggregateConf getAggregateConf() { - return slider.getAppState().getInstanceDefinitionSnapshot(); + //TODO + return new AggregateConf(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java index 2f02f274456..79b687fb51d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java @@ -49,7 +49,7 @@ public class ClusterSpecificationBlock extends SliderHamletBlock { * @return */ private String getJson() { - return appState.getClusterStatus().toString(); + return appState.getApplication().toString(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java index 8b7d69515a7..4796d6c6c2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java @@ -26,8 +26,8 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR; -import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.ClusterNode; +import org.apache.slider.api.resource.Application; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.server.appmaster.state.RoleInstance; import org.apache.slider.server.appmaster.web.WebAppApi; @@ -141,18 +141,10 @@ public class ContainerStatsBlock extends SliderHamletBlock { })); - ClusterDescription desc = appState.getClusterStatus(); - Map options = desc.getRole(name); + Application application = appState.getApplication(); Iterable> tableContent; - - // Generate the pairs of data in the expected form - if (null != options) { - tableContent = Iterables.transform(options.entrySet(), stringStringPairFunc); - } else { - // Or catch that we have no options and provide "empty" - tableContent = Collections.emptySet(); - } - + tableContent = Collections.emptySet(); + // Generate the options used by this role generateRoleDetails(div, "role-options-wrap", "Role Options", tableContent); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java index 2f99b279872..440094e2959 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java @@ -22,15 +22,12 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.LI; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL; -import org.apache.slider.api.ClusterDescription; -import org.apache.slider.api.StatusKeys; import org.apache.slider.api.types.ApplicationLivenessInformation; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.core.registry.docstore.ExportEntry; import org.apache.slider.core.registry.docstore.PublishedExports; import org.apache.slider.core.registry.docstore.PublishedExportsSet; -import org.apache.slider.providers.MonitorDetail; -import org.apache.slider.providers.ProviderService; +import org.apache.slider.server.appmaster.metrics.SliderMetrics; import org.apache.slider.server.appmaster.state.RoleStatus; import org.apache.slider.server.appmaster.web.WebAppApi; import org.slf4j.Logger; @@ -39,7 +36,6 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Map.Entry; import java.util.Set; @@ -71,8 +67,7 @@ public class IndexBlock extends SliderHamletBlock { // An extra method to make testing easier since you can't make an instance of Block @VisibleForTesting protected void doIndex(Hamlet html, String providerName) { - ClusterDescription clusterStatus = appState.getClusterStatus(); - String name = clusterStatus.name; + String name = appState.getApplicationName(); if (name != null && (name.startsWith(" ") || name.endsWith(" "))) { name = "'" + name + "'"; } @@ -96,23 +91,23 @@ public class IndexBlock extends SliderHamletBlock { ._(); table1.tr() .td("Create time: ") - .td(getInfoAvoidingNulls(StatusKeys.INFO_CREATE_TIME_HUMAN)) + .td("N/A") ._(); table1.tr() .td("Running since: ") - .td(getInfoAvoidingNulls(StatusKeys.INFO_LIVE_TIME_HUMAN)) + .td("N/A") ._(); table1.tr() .td("Time last flexed: ") - .td(getInfoAvoidingNulls(StatusKeys.INFO_FLEX_TIME_HUMAN)) + .td("N/A") ._(); table1.tr() .td("Application storage path: ") - .td(clusterStatus.dataPath) + .td("N/A") ._(); table1.tr() .td("Application configuration path: ") - .td(clusterStatus.originConfigurationPath) + .td("N/A") ._(); table1._(); div._(); @@ -136,7 +131,8 @@ public class IndexBlock extends SliderHamletBlock { trb(header, "Placement"); header._()._(); // tr & thead - List roleStatuses = appState.cloneRoleStatusList(); + List roleStatuses = + new ArrayList<>(appState.getRoleStatusMap().values()); Collections.sort(roleStatuses, new RoleStatus.CompareByName()); for (RoleStatus status : roleStatuses) { String roleName = status.getName(); @@ -144,7 +140,7 @@ public class IndexBlock extends SliderHamletBlock { String aatext; if (status.isAntiAffinePlacement()) { boolean aaRequestOutstanding = status.isAARequestOutstanding(); - int pending = (int)status.getPendingAntiAffineRequests(); + int pending = (int)status.getAAPending(); aatext = buildAADetails(aaRequestOutstanding, pending); if (SliderUtils.isSet(status.getLabelExpression())) { aatext += " (label: " + status.getLabelExpression() + ")"; @@ -160,17 +156,17 @@ public class IndexBlock extends SliderHamletBlock { } else { aatext = ""; } - if (status.getRequested() > 0) { + if (status.getPending() > 0) { roleWithOpenRequest ++; } } + SliderMetrics metrics = status.getComponentMetrics(); table.tr() .td().a(nameUrl, roleName)._() - .td(String.format("%d", status.getDesired())) - .td(String.format("%d", status.getActual())) - .td(String.format("%d", status.getRequested())) - .td(String.format("%d", status.getFailed())) - .td(String.format("%d", status.getStartFailed())) + .td(String.format("%d", metrics.containersDesired.value())) + .td(String.format("%d", metrics.containersRunning.value())) + .td(String.format("%d", metrics.containersPending.value())) + .td(String.format("%d", metrics.containersFailed.value())) .td(aatext) ._(); } @@ -218,7 +214,7 @@ public class IndexBlock extends SliderHamletBlock { DIV provider_info = html.div("provider_info"); provider_info.h3(providerName + " information"); UL ul = html.ul(); - addProviderServiceOptions(providerService, ul, clusterStatus); + //TODO render app/cluster status ul._(); provider_info._(); @@ -250,40 +246,9 @@ public class IndexBlock extends SliderHamletBlock { } private String getProviderName() { - return providerService.getHumanName(); + return "docker"; } - private String getInfoAvoidingNulls(String key) { - String createTime = appState.getClusterStatus().getInfo(key); - - return null == createTime ? "N/A" : createTime; - } - - protected void addProviderServiceOptions(ProviderService provider, - UL ul, ClusterDescription clusterStatus) { - Map details = provider.buildMonitorDetails( - clusterStatus); - if (null == details) { - return; - } - // Loop over each entry, placing the text in the UL, adding an anchor when the URL is non-null/empty - for (Entry entry : details.entrySet()) { - MonitorDetail detail = entry.getValue(); - if (SliderUtils.isSet(detail.getValue()) ) { - LI item = ul.li(); - item.span().$class("bold")._(entry.getKey())._(); - item._(" - "); - if (detail.isUrl()) { - // Render an anchor if the value is a URL - item.a(detail.getValue(), detail.getValue())._(); - } else { - item._(detail.getValue())._(); - } - } else { - ul.li(entry.getKey()); - } - } - } protected void enumeratePublishedExports(PublishedExportsSet exports, UL ul) { for(String key : exports.keys()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java deleted file mode 100644 index 92df0487ea0..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.servicemonitor; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.slider.client.SliderYarnClientImpl; -import org.apache.slider.core.exceptions.UnknownApplicationInstanceException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.List; - -/** - * Probe for YARN application - */ -public class YarnApplicationProbe extends Probe { - protected static final Logger log = LoggerFactory.getLogger( - YarnApplicationProbe.class); - - /** - * Yarn client service - */ - private SliderYarnClientImpl yarnClient; - private final String clustername; - private final String username; - - public YarnApplicationProbe(String clustername, - SliderYarnClientImpl yarnClient, - String name, - Configuration conf, String username) - throws IOException { - super("Port probe " + name + " " + clustername, - conf); - this.clustername = clustername; - this.yarnClient = yarnClient; - this.username = username; - } - - - @Override - public void init() throws IOException { - - log.info("Checking " + clustername ); - } - - /** - * Try to connect to the (host,port); a failure to connect within - * the specified timeout is a failure - * @param livePing is the ping live: true for live; false for boot time - * @return the outcome - */ - @Override - public ProbeStatus ping(boolean livePing) { - ProbeStatus status = new ProbeStatus(); - try { - List instances = yarnClient - .listDeployedInstances(username, null, clustername); - ApplicationReport instance = yarnClient - .findClusterInInstanceList(instances, clustername); - if (null == instance) { - throw UnknownApplicationInstanceException.unknownInstance(clustername); - } - status.succeed(this); - } catch (Exception e) { - status.fail(this, e); - } - return status; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java index 254bf278d79..6defa2b6925 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java @@ -210,8 +210,6 @@ public class YarnRegistryViewForProviders { /** * Add a service under a path for the current user - * @param serviceClass service class to use under ~user - * @param serviceName name of the service * @param record service record * @param deleteTreeFirst perform recursive delete of the path first * @return the path the service was created at diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java similarity index 98% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java index 23b7ad4711e..7e8cf5ba6ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.services.utils; +package org.apache.slider.util; public interface RestApiConstants { String CONTEXT_ROOT = "/services/v1"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java similarity index 98% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java index 2d739a42b1c..0f6247d04b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.hadoop.yarn.services.utils; +package org.apache.slider.util; public interface RestApiErrorMessages { String ERROR_APPLICATION_NAME_INVALID = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java new file mode 100644 index 00000000000..776ce003d0a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/ServiceApiUtil.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.util; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.StringUtils; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Artifact; +import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.Configuration; +import org.apache.slider.api.resource.Resource; +import org.apache.slider.common.tools.SliderUtils; + +public class ServiceApiUtil { + + @VisibleForTesting + public static void validateApplicationPostPayload(Application application) { + if (StringUtils.isEmpty(application.getName())) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID); + } + if (!SliderUtils.isClusternameValid(application.getName())) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID_FORMAT); + } + + // If the application has no components do top-level checks + if (!hasComponent(application)) { + // artifact + if (application.getArtifact() == null) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_ARTIFACT_INVALID); + } + if (StringUtils.isEmpty(application.getArtifact().getId())) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID); + } + + // If artifact is of type APPLICATION, add a slider specific property + if (application.getArtifact().getType() + == Artifact.TypeEnum.APPLICATION) { + if (application.getConfiguration() == null) { + application.setConfiguration(new Configuration()); + } + } + // resource + validateApplicationResource(application.getResource(), null, + application.getArtifact().getType()); + + // container size + if (application.getNumberOfContainers() == null) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_CONTAINERS_COUNT_INVALID); + } + + // Since it is a simple app with no components, create a default component + application.getComponents().add(createDefaultComponent(application)); + } else { + // If the application has components, then run checks for each component. + // Let global values take effect if component level values are not + // provided. + Artifact globalArtifact = application.getArtifact(); + Resource globalResource = application.getResource(); + Long globalNumberOfContainers = application.getNumberOfContainers(); + for (Component comp : application.getComponents()) { + // artifact + if (comp.getArtifact() == null) { + comp.setArtifact(globalArtifact); + } + // If still null raise validation exception + if (comp.getArtifact() == null) { + throw new IllegalArgumentException(String + .format(RestApiErrorMessages.ERROR_ARTIFACT_FOR_COMP_INVALID, + comp.getName())); + } + if (StringUtils.isEmpty(comp.getArtifact().getId())) { + throw new IllegalArgumentException(String + .format(RestApiErrorMessages.ERROR_ARTIFACT_ID_FOR_COMP_INVALID, + comp.getName())); + } + + // If artifact is of type APPLICATION, add a slider specific property + if (comp.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) { + if (comp.getConfiguration() == null) { + comp.setConfiguration(new Configuration()); + } + comp.setName(comp.getArtifact().getId()); + } + + // resource + if (comp.getResource() == null) { + comp.setResource(globalResource); + } + validateApplicationResource(comp.getResource(), comp, + comp.getArtifact().getType()); + + // container count + if (comp.getNumberOfContainers() == null) { + comp.setNumberOfContainers(globalNumberOfContainers); + } + if (comp.getNumberOfContainers() == null) { + throw new IllegalArgumentException(String.format( + RestApiErrorMessages.ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID, + comp.getName())); + } + } + } + + // Application lifetime if not specified, is set to unlimited lifetime + if (application.getLifetime() == null) { + application.setLifetime(RestApiConstants.DEFAULT_UNLIMITED_LIFETIME); + } + } + + private static void validateApplicationResource(Resource resource, + Component comp, Artifact.TypeEnum artifactType) { + // Only apps/components of type APPLICATION can skip resource requirement + if (resource == null && artifactType == Artifact.TypeEnum.APPLICATION) { + return; + } + if (resource == null) { + throw new IllegalArgumentException( + comp == null ? RestApiErrorMessages.ERROR_RESOURCE_INVALID : String + .format(RestApiErrorMessages.ERROR_RESOURCE_FOR_COMP_INVALID, + comp.getName())); + } + // One and only one of profile OR cpus & memory can be specified. Specifying + // both raises validation error. + if (StringUtils.isNotEmpty(resource.getProfile()) && ( + resource.getCpus() != null || StringUtils + .isNotEmpty(resource.getMemory()))) { + throw new IllegalArgumentException(comp == null ? + RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED : + String.format( + RestApiErrorMessages.ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED, + comp.getName())); + } + // Currently resource profile is not supported yet, so we will raise + // validation error if only resource profile is specified + if (StringUtils.isNotEmpty(resource.getProfile())) { + throw new IllegalArgumentException( + RestApiErrorMessages.ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET); + } + + String memory = resource.getMemory(); + Integer cpus = resource.getCpus(); + if (StringUtils.isEmpty(memory)) { + throw new IllegalArgumentException( + comp == null ? RestApiErrorMessages.ERROR_RESOURCE_MEMORY_INVALID : + String.format( + RestApiErrorMessages.ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID, + comp.getName())); + } + if (cpus == null) { + throw new IllegalArgumentException( + comp == null ? RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID : + String.format( + RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID, + comp.getName())); + } + if (cpus <= 0) { + throw new IllegalArgumentException(comp == null ? + RestApiErrorMessages.ERROR_RESOURCE_CPUS_INVALID_RANGE : String + .format( + RestApiErrorMessages.ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE, + comp.getName())); + } + } + + public static boolean hasComponent(Application application) { + if (application.getComponents() == null || application.getComponents() + .isEmpty()) { + return false; + } + return true; + } + + public static Component createDefaultComponent(Application app) { + Component comp = new Component(); + comp.setName(RestApiConstants.DEFAULT_COMPONENT_NAME); + comp.setArtifact(app.getArtifact()); + comp.setResource(app.getResource()); + comp.setNumberOfContainers(app.getNumberOfContainers()); + comp.setLaunchCommand(app.getLaunchCommand()); + return comp; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto index b8bdc591a42..bfcab23ae4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto @@ -80,22 +80,14 @@ message UpgradeContainersRequestProto { message UpgradeContainersResponseProto { } -/** - * flex the cluster - */ -message FlexClusterRequestProto { - required string clusterSpec = 1; +message FlexComponentRequestProto { + optional string name = 1; + optional int32 numberOfContainers = 2; } - -/** - * flex the cluster - */ -message FlexClusterResponseProto { - required bool response = 1; +message FlexComponentResponseProto { } - /** * void request */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto index d68da2bbfb9..8a0faf94fe5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto @@ -61,24 +61,14 @@ service SliderClusterProtocolPB { rpc upgradeContainers(UpgradeContainersRequestProto) returns(UpgradeContainersResponseProto); - /** - * Flex the cluster. - */ - rpc flexCluster(FlexClusterRequestProto) - returns(FlexClusterResponseProto); + rpc flexComponent(FlexComponentRequestProto) returns (FlexComponentResponseProto); /** * Get the current cluster status */ rpc getJSONClusterStatus(GetJSONClusterStatusRequestProto) returns(GetJSONClusterStatusResponseProto); - - /** - * Get the instance definition - */ - rpc getInstanceDefinition(GetInstanceDefinitionRequestProto) - returns(GetInstanceDefinitionResponseProto); - + /** * List all running nodes in a role */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml index a2517d5c3fb..96bfe0fc081 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml @@ -23,10 +23,6 @@ slider.config.loaded true - - slider.provider.agent - org.apache.slider.providers.agent.AgentProviderFactory - slider.provider.docker org.apache.slider.providers.docker.DockerProviderFactory diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java deleted file mode 100644 index b9559313278..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java +++ /dev/null @@ -1,157 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import java.lang.reflect.Method; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.LogAggregationContext; -import org.apache.hadoop.yarn.client.api.YarnClientApplication; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.client.SliderYarnClientImpl; -import org.apache.slider.common.SliderKeys; -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class TestAppMasterLauncher { - SliderYarnClientImpl mockYarnClient; - YarnClientApplication yarnClientApp; - ApplicationSubmissionContext appSubmissionContext; - Set tags = Collections.emptySet(); - AppMasterLauncher appMasterLauncher = null; - boolean isOldApi = true; - Method rolledLogsIncludeMethod = null; - Method rolledLogsExcludeMethod = null; - - @Before - public void initialize() throws Exception { - mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class); - yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class); - appSubmissionContext = EasyMock - .createNiceMock(ApplicationSubmissionContext.class); - EasyMock.expect(yarnClientApp.getApplicationSubmissionContext()) - .andReturn(appSubmissionContext).once(); - EasyMock.expect(mockYarnClient.createApplication()) - .andReturn(yarnClientApp).once(); - - try { - LogAggregationContext.class.getMethod("newInstance", String.class, - String.class, String.class, String.class); - isOldApi = false; - rolledLogsIncludeMethod = LogAggregationContext.class - .getMethod("getRolledLogsIncludePattern"); - rolledLogsExcludeMethod = LogAggregationContext.class - .getMethod("getRolledLogsExcludePattern"); - } catch (Exception e) { - isOldApi = true; - } - } - - /** - * These tests will probably fail when compiled against hadoop 2.7+. Please - * refer to SLIDER-810. It has been purposely not modified so that it fails - * and that someone needs to modify the code in - * {@code AbstractLauncher#extractLogAggregationContext(Map)}. Comments are - * provided in that method as to what needs to be done. - * - * @throws Exception - */ - @Test - public void testExtractLogAggregationContext() throws Exception { - Map options = new HashMap(); - options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, - " | slider*.txt |agent.out| |"); - options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, - "command*.json| agent.log* | "); - - EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); - appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null, - null, mockYarnClient, false, null, options, tags, null); - - // Verify the include/exclude patterns - String expectedInclude = "slider*.txt|agent.out"; - String expectedExclude = "command*.json|agent.log*"; - assertPatterns(expectedInclude, expectedExclude); - - EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); - - } - - @Test - public void testExtractLogAggregationContextEmptyIncludePattern() - throws Exception { - Map options = new HashMap(); - options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, " "); - options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, - "command*.json| agent.log* | "); - - EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); - appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null, - null, mockYarnClient, false, null, options, tags, null); - - // Verify the include/exclude patterns - String expectedInclude = isOldApi ? "" : ".*"; - String expectedExclude = "command*.json|agent.log*"; - assertPatterns(expectedInclude, expectedExclude); - - EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); - } - - @Test - public void testExtractLogAggregationContextEmptyIncludeAndExcludePattern() - throws Exception { - Map options = new HashMap(); - options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, ""); - options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, " "); - - EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp); - appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null, - null, mockYarnClient, false, null, options, tags, null); - - // Verify the include/exclude patterns - String expectedInclude = isOldApi ? "" : ".*"; - String expectedExclude = ""; - assertPatterns(expectedInclude, expectedExclude); - - EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp); - } - - private void assertPatterns(String expectedIncludePattern, - String expectedExcludePattern) throws Exception { - if (isOldApi) { - Assert.assertEquals(expectedIncludePattern, - appMasterLauncher.logAggregationContext.getIncludePattern()); - Assert.assertEquals(expectedExcludePattern, - appMasterLauncher.logAggregationContext.getExcludePattern()); - } else { - Assert.assertEquals(expectedIncludePattern, - (String) rolledLogsIncludeMethod - .invoke(appMasterLauncher.logAggregationContext)); - Assert.assertEquals(expectedExcludePattern, - (String) rolledLogsExcludeMethod - .invoke(appMasterLauncher.logAggregationContext)); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java deleted file mode 100644 index a8f6b266ef4..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.core.launch; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.client.api.YarnClientApplication; -import org.apache.hadoop.yarn.util.Records; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.client.SliderYarnClientImpl; -import org.apache.slider.common.SliderKeys; -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class TestAppMasterLauncherWithAmReset { - SliderYarnClientImpl mockYarnClient; - YarnClientApplication yarnClientApp; - ApplicationSubmissionContext appSubmissionContext; - GetNewApplicationResponse newApp; - Set tags = Collections.emptySet(); - AppMasterLauncher appMasterLauncher = null; - boolean isOldApi = true; - - @Before - public void initialize() throws Exception { - mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class); - yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class); - newApp = EasyMock.createNiceMock(GetNewApplicationResponse.class); - EasyMock.expect(mockYarnClient.createApplication()) - .andReturn(new YarnClientApplication(newApp, - Records.newRecord(ApplicationSubmissionContext.class))); - } - - @Test - public void testExtractYarnResourceManagerAmRetryCountWindowMs() throws - Exception { - Map options = new HashMap(); - final String expectedInterval = Integer.toString (120000); - options.put(ResourceKeys.YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS, - expectedInterval); - EasyMock.replay(mockYarnClient, yarnClientApp); - - appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null, - null, mockYarnClient, false, null, options, tags, null); - - ApplicationSubmissionContext ctx = appMasterLauncher.application - .getApplicationSubmissionContext(); - String retryIntervalWindow = Long.toString(ctx - .getAttemptFailuresValidityInterval()); - Assert.assertEquals(expectedInterval, retryIntervalWindow); - } - - @Test - public void testExtractYarnResourceManagerAmRetryCountWindowMsDefaultValue() - throws Exception { - Map options = new HashMap(); - EasyMock.replay(mockYarnClient, yarnClientApp); - - appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null, - null, mockYarnClient, false, null, options, tags, null); - - ApplicationSubmissionContext ctx = appMasterLauncher.application - .getApplicationSubmissionContext(); - long retryIntervalWindow = ctx.getAttemptFailuresValidityInterval(); - Assert.assertEquals(ResourceKeys.DEFAULT_AM_RETRY_COUNT_WINDOW_MS, - retryIntervalWindow); - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java deleted file mode 100644 index a1986cd3bb8..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster; - -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.core.conf.MapOperations; -import org.junit.Assert; -import org.junit.Test; - -import java.util.HashMap; -import java.util.Map; - -/** - * - */ -public class TestServiceRecordAttributes extends Assert { - - @Test - public void testAppConfigProvidedServiceRecordAttributes() throws Exception { - Map options = new HashMap<>(); - options.put("slider.some.arbitrary.option", "arbitrary value"); - options.put("service.record.attribute.one_attribute", "one_attribute_value"); - options.put("service.record.attribute.second_attribute", "second_attribute_value"); - MapOperations serviceProps = new MapOperations(SliderKeys.COMPONENT_AM, options); - options = new HashMap<>(); - options.put("some.component.attribute", "component_attribute_value"); - options.put("service.record.attribute.component_attribute", "component_attribute_value"); - MapOperations compProps = new MapOperations("TEST_COMP", options); - - SliderAppMaster appMaster = new SliderAppMaster(); - - ServiceRecord appServiceRecord = new ServiceRecord(); - - appMaster.setProvidedServiceRecordAttributes(serviceProps, appServiceRecord); - - assertNull("property should not be attribute", - appServiceRecord.get("slider.some.arbitrary.option")); - assertEquals("wrong value", "one_attribute_value", - appServiceRecord.get("one_attribute")); - assertEquals("wrong value", "second_attribute_value", - appServiceRecord.get("second_attribute")); - - ServiceRecord compServiceRecord = new ServiceRecord(); - - appMaster.setProvidedServiceRecordAttributes(compProps, compServiceRecord); - - assertNull("should not be attribute", - compServiceRecord.get("some.component.attribute")); - assertEquals("wrong value", "component_attribute_value", - compServiceRecord.get("component_attribute")); - - } -}