Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-6160. Create an agent-less docker-less provider in the native services framework. Contributed by Billie Rinaldi

This commit is contained in:
Jian He 2017-05-17 09:58:32 -07:00
parent a22256bd40
commit 9085cd5c6e
28 changed files with 664 additions and 538 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
View File

@ -133,7 +133,6 @@ import org.apache.slider.core.zk.BlockingZKWatcher;
import org.apache.slider.core.zk.ZKIntegration;
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.providers.SliderProviderFactory;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.rpc.RpcBinder;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
@ -257,8 +256,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
config = super.bindArgs(config, args);
serviceArgs = new ClientArgs(args);
serviceArgs.parse();
// add the slider XML config
ConfigHelper.injectSliderXMLResource();
// yarn-ify
YarnConfiguration yarnConfiguration = new YarnConfiguration(config);
return patchConfiguration(yarnConfiguration);
@ -622,14 +619,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
@Override
public AbstractClientProvider createClientProvider(String provider)
throws SliderException {
SliderProviderFactory factory =
SliderProviderFactory.createSliderProviderFactory(provider);
return factory.createClientProvider();
}
private Application getApplicationFromArgs(String clusterName,
AbstractClusterBuildingActionArgs args) throws IOException {
File file = args.getAppDef();
@ -893,7 +882,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private void persistApp(Path appDir, Application application)
throws IOException, SliderException {
FsPermission appDirPermission = new FsPermission("777");
FsPermission appDirPermission = new FsPermission("750");
sliderFileSystem.createWithPermissions(appDir, appDirPermission);
Path appJson = new Path(appDir, application.getName() + ".json");
jsonSerDeser
@ -1190,17 +1179,18 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
// TODO handle client install
// Only INSTALL is supported
AbstractClientProvider
provider = createClientProvider(SliderProviderFactory.DEFAULT_CLUSTER_TYPE);
provider.processClientOperation(sliderFileSystem,
getRegistryOperations(),
getConfig(),
"INSTALL",
clientInfo.installLocation,
pkgFile,
config,
clientInfo.name);
// ClientProvider
// provider = createClientProvider(SliderProviderFactory.DEFAULT_CLUSTER_TYPE);
// provider.processClientOperation(sliderFileSystem,
// getRegistryOperations(),
// getConfig(),
// "INSTALL",
// clientInfo.installLocation,
// pkgFile,
// config,
// clientInfo.name);
return EXIT_SUCCESS;
}

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
View File

@ -42,7 +42,6 @@ import org.apache.slider.common.params.ActionThawArgs;
import org.apache.slider.common.params.ActionUpgradeArgs;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.providers.AbstractClientProvider;
import java.io.IOException;
@ -62,15 +61,6 @@ public interface SliderClientAPI extends Service {
int actionAmSuicide(String clustername,
ActionAMSuicideArgs args) throws YarnException, IOException;
/**
* Get the provider for this cluster
* @param provider the name of the provider
* @return the provider instance
* @throws SliderException problems building the provider
*/
AbstractClientProvider createClientProvider(String provider)
throws SliderException;
/**
* Manage keytabs leveraged by slider
*

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
View File

@ -18,8 +18,6 @@
package org.apache.slider.common;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@ -158,11 +156,6 @@ public interface SliderKeys extends SliderXmlConfKeys {
*/
String HISTORY_FILENAME_GLOB_PATTERN = HISTORY_FILENAME_PREFIX +"*."+
HISTORY_FILENAME_SUFFIX;
/**
* XML resource listing the standard Slider providers
* {@value}
*/
String SLIDER_XML = "org/apache/slider/slider.xml";
String CLUSTER_DIRECTORY = "cluster";
@ -311,5 +304,12 @@ public interface SliderKeys extends SliderXmlConfKeys {
String APP_RESOURCES = "application.resources";
String APP_RESOURCES_DIR = "app/resources";
String APP_PACKAGES_DIR = "app/packages";
String APP_INSTALL_DIR = "app/install";
String OUT_FILE = "stdout.txt";
String ERR_FILE = "stderr.txt";
String QUICK_LINKS = "quicklinks";
String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
}

39
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
View File

@ -50,7 +50,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* Methods to aid in config, both in the Configuration class and
@ -62,11 +61,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
public class ConfigHelper {
private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class);
private static AtomicBoolean sliderResourceInjected =
new AtomicBoolean(false);
private static AtomicBoolean sliderResourceInjectionAttempted =
new AtomicBoolean(false);
/**
* Dump the (sorted) configuration
* @param conf config
@ -614,37 +608,4 @@ public class ConfigHelper {
public static void registerDeprecatedConfigItems() {
}
/**
* Load a configuration with the {@link SliderKeys#SLIDER_XML} resource
* included
* @return a configuration instance
*/
public static Configuration loadSliderConfiguration() {
Configuration conf = new Configuration();
conf.addResource(SliderKeys.SLIDER_XML);
return conf;
}
/**
* Inject the {@link SliderKeys#SLIDER_XML} resource
* into the configuration resources <i>of all configurations</i>.
* <p>
* This operation is idempotent.
* <p>
* If the resource is not on the classpath, downgrades, rather than
* fails.
* @return true if the resource was found and loaded.
*/
public static synchronized boolean injectSliderXMLResource() {
if (sliderResourceInjectionAttempted.getAndSet(true)) {
return sliderResourceInjected.get();
}
URL resourceUrl = getResourceUrl(SliderKeys.SLIDER_XML);
if (resourceUrl != null) {
Configuration.addDefaultResource(SliderKeys.SLIDER_XML);
sliderResourceInjected.set(true);
}
return sliderResourceInjected.get();
}
}

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
View File

@ -19,32 +19,26 @@
package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.SliderException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
public abstract class AbstractClientProvider extends Configured {
private static final Logger log =
LoggerFactory.getLogger(AbstractClientProvider.class);
public abstract class AbstractClientProvider {
public AbstractClientProvider(Configuration conf) {
super(conf);
public AbstractClientProvider() {
}
public abstract String getName();
public abstract List<ProviderRole> getRoles();
/**
* Generates a fixed format of application tags given one or more of
* application name, version and description. This allows subsequent query for
@ -70,7 +64,29 @@ public abstract class AbstractClientProvider extends Configured {
}
/**
* Process client operations for applications such as install, configure
* Validate the artifact.
* @param artifact
*/
public abstract void validateArtifact(Artifact artifact, FileSystem
fileSystem) throws IOException;
protected abstract void validateConfigFile(ConfigFile configFile, FileSystem
fileSystem) throws IOException;
/**
* Validate the config files.
* @param configFiles config file list
* @param fileSystem file system
*/
public void validateConfigFiles(List<ConfigFile> configFiles, FileSystem
fileSystem) throws IOException {
for (ConfigFile configFile : configFiles) {
validateConfigFile(configFile, fileSystem);
}
}
/**
* Process client operations for applications such as install, configure.
* @param fileSystem
* @param registryOperations
* @param configuration

169
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java Normal file
View File

@ -0,0 +1,169 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.Component;
import org.apache.slider.api.resource.ContainerState;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.CommandLineBuilder;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Map;
import java.util.Map.Entry;
import static org.apache.slider.util.ServiceApiUtil.$;
public abstract class AbstractProviderService extends AbstractService
implements ProviderService, SliderKeys {
protected static final Logger log =
LoggerFactory.getLogger(AbstractProviderService.class);
private static final ProviderUtils providerUtils = new ProviderUtils(log);
protected StateAccessForProviders amState;
protected YarnRegistryViewForProviders yarnRegistry;
private ServiceTimelinePublisher serviceTimelinePublisher;
protected AbstractProviderService(String name) {
super(name);
}
public abstract void processArtifact(ContainerLauncher launcher, Component
component, SliderFileSystem fileSystem) throws IOException;
@Override
public void setAMState(StateAccessForProviders stateAccessor) {
this.amState = stateAccessor;
}
@Override
public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) {
this.yarnRegistry = yarnRegistry;
}
public void buildContainerLaunchContext(ContainerLauncher launcher,
Application application, Container container, ProviderRole providerRole,
SliderFileSystem fileSystem, RoleInstance roleInstance)
throws IOException, SliderException {
Component component = providerRole.component;
processArtifact(launcher, component, fileSystem);
// Generate tokens (key-value pair) for config substitution.
// Get pre-defined tokens
Map<String, String> globalTokens = amState.getGlobalSubstitutionTokens();
Map<String, String> tokensForSubstitution = providerUtils
.initCompTokensForSubstitute(roleInstance);
tokensForSubstitution.putAll(globalTokens);
// Set the environment variables in launcher
launcher.putEnv(SliderUtils
.buildEnvMap(component.getConfiguration(), tokensForSubstitution));
launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
if (System.getenv(HADOOP_USER_NAME) != null) {
launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
}
launcher.setEnv("LANG", "en_US.UTF-8");
launcher.setEnv("LC_ALL", "en_US.UTF-8");
launcher.setEnv("LANGUAGE", "en_US.UTF-8");
for (Entry<String, String> entry : launcher.getEnv().entrySet()) {
tokensForSubstitution.put($(entry.getKey()), entry.getValue());
}
providerUtils.addComponentHostTokens(tokensForSubstitution, amState);
// create config file on hdfs and add local resource
providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
component, tokensForSubstitution, roleInstance, amState);
// substitute launch command
String launchCommand = ProviderUtils
.substituteStrWithTokens(component.getLaunchCommand(),
tokensForSubstitution);
CommandLineBuilder operation = new CommandLineBuilder();
operation.add(launchCommand);
operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
launcher.addCommand(operation.build());
// publish exports
providerUtils
.substituteMapWithTokens(application.getQuicklinks(), tokensForSubstitution);
PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS,
application.getQuicklinks().entrySet());
amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf);
if (serviceTimelinePublisher != null) {
serviceTimelinePublisher.serviceAttemptUpdated(application);
}
}
public boolean processContainerStatus(ContainerId containerId,
ContainerStatus status) {
log.debug("Handling container status: {}", status);
if (SliderUtils.isEmpty(status.getIPs()) ||
SliderUtils.isUnset(status.getHost())) {
return true;
}
RoleInstance instance = amState.getOwnedContainer(containerId);
if (instance == null) {
// container is completed?
return false;
}
try {
providerUtils.updateServiceRecord(amState, yarnRegistry,
containerId.toString(), instance.role, status.getIPs(), status.getHost());
} catch (IOException e) {
// could not write service record to ZK, log and retry
log.warn("Error updating container {} service record in registry, " +
"retrying", containerId, e);
return true;
}
// TODO publish ip and host
org.apache.slider.api.resource.Container container =
instance.providerRole.component.getContainer(containerId.toString());
if (container != null) {
container.setIp(StringUtils.join(",", status.getIPs()));
container.setHostname(status.getHost());
container.setState(ContainerState.READY);
} else {
log.warn(containerId + " not found in Application!");
}
return false;
}
@Override
public void setServiceTimelinePublisher(ServiceTimelinePublisher publisher) {
this.serviceTimelinePublisher = publisher;
}
}

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers;
import org.apache.hadoop.fs.FileSystem;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.api.resource.ConfigFile;
import java.io.IOException;
import java.nio.file.Paths;
public class DefaultClientProvider extends AbstractClientProvider {
public DefaultClientProvider() {
}
@Override
public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
}
@Override
protected void validateConfigFile(ConfigFile configFile, FileSystem
fileSystem) throws IOException {
// validate dest_file is not absolute
if (Paths.get(configFile.getDestFile()).isAbsolute()) {
throw new IllegalArgumentException(
"Dest_file must not be absolute path: " + configFile.getDestFile());
}
}
}

47
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java Normal file
View File

@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers;
public final class DefaultProviderFactory extends SliderProviderFactory {
private static final SliderProviderFactory FACTORY = new
DefaultProviderFactory();
private DefaultProviderFactory() {}
private static class Client {
static final AbstractClientProvider PROVIDER = new DefaultClientProvider();
}
private static class Server {
static final ProviderService PROVIDER = new DefaultProviderService();
}
@Override
public AbstractClientProvider createClientProvider() {
return Client.PROVIDER;
}
@Override
public ProviderService createServerProvider() {
return Server.PROVIDER;
}
public static SliderProviderFactory getInstance() {
return FACTORY;
}
}

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers;
import org.apache.slider.api.resource.Component;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.providers.AbstractProviderService;
import java.io.IOException;
public class DefaultProviderService extends AbstractProviderService {
protected DefaultProviderService() {
super(DefaultProviderService.class.getSimpleName());
}
@Override
public void processArtifact(ContainerLauncher launcher, Component
component, SliderFileSystem fileSystem) throws IOException {
}
}

105
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
View File

@ -18,93 +18,60 @@
package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.providers.docker.DockerProviderFactory;
import org.apache.slider.providers.tarball.TarballProviderFactory;
import org.apache.slider.util.RestApiErrorMessages;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Base class for factories
* Base class for factories.
*/
public abstract class SliderProviderFactory extends Configured {
public static final String DEFAULT_CLUSTER_TYPE = AgentKeys.PROVIDER_AGENT;
public abstract class SliderProviderFactory {
protected static final Logger log =
LoggerFactory.getLogger(SliderProviderFactory.class);
public static final String PROVIDER_NOT_FOUND =
"Unable to find provider of application type %s";
LoggerFactory.getLogger(SliderProviderFactory.class);
public SliderProviderFactory(Configuration conf) {
super(conf);
}
protected SliderProviderFactory() {
}
protected SliderProviderFactory() {}
public abstract AbstractClientProvider createClientProvider();
public abstract ProviderService createServerProvider();
public static synchronized ProviderService getProviderService(Artifact
artifact) {
return createSliderProviderFactory(artifact).createServerProvider();
}
public static synchronized AbstractClientProvider getClientProvider(Artifact
artifact) {
return createSliderProviderFactory(artifact).createClientProvider();
}
/**
* Create a provider for a specific application
* @param application app
* @return app instance
* @param artifact artifact
* @return provider factory
* @throws SliderException on any instantiation problem
*/
public static SliderProviderFactory createSliderProviderFactory(String application) throws
SliderException {
Configuration conf = loadSliderConfiguration();
if (application == null) {
application = DEFAULT_CLUSTER_TYPE;
public static synchronized SliderProviderFactory createSliderProviderFactory(
Artifact artifact) {
if (artifact == null || artifact.getType() == null) {
log.info("Loading service provider type default");
return DefaultProviderFactory.getInstance();
}
String providerKey =
String.format(SliderXmlConfKeys.KEY_PROVIDER, application);
if (application.contains(".")) {
log.debug("Treating {} as a classname", application);
String name = "classname.key";
conf.set(name, application);
providerKey = name;
log.info("Loading service provider type {}", artifact.getType());
switch (artifact.getType()) {
// TODO add handling for custom types?
// TODO handle application
case DOCKER:
return DockerProviderFactory.getInstance();
case TARBALL:
return TarballProviderFactory.getInstance();
default:
throw new IllegalArgumentException(
RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
}
Class<? extends SliderProviderFactory> providerClass;
try {
providerClass = conf.getClass(providerKey, null, SliderProviderFactory.class);
} catch (RuntimeException e) {
throw new BadClusterStateException(e, "Failed to load provider %s: %s", application, e);
}
if (providerClass == null) {
throw new BadClusterStateException(PROVIDER_NOT_FOUND, application);
}
Exception ex;
try {
SliderProviderFactory providerFactory = providerClass.newInstance();
providerFactory.setConf(conf);
return providerFactory;
} catch (Exception e) {
ex = e;
}
//by here the operation failed and ex is set to the value
throw new BadClusterStateException(ex,
"Failed to create an instance of %s : %s",
providerClass,
ex);
}
/**
* Load a configuration with the {@link SliderKeys#SLIDER_XML} resource
* included
* @return a configuration instance
*/
public static Configuration loadSliderConfiguration() {
Configuration conf = new Configuration();
conf.addResource(SliderKeys.SLIDER_XML);
return conf;
}
}

102
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
View File

@ -1,102 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
/*
*/
public interface AgentKeys {
String AGENT_TAR = "slider-agent.tar.gz";
String PROVIDER_AGENT = "agent";
String ROLE_NODE = "echo";
/**
* Template stored in the slider classpath -to use if there is
* no site-specific template
* {@value}
*/
String CONF_RESOURCE = "org/apache/slider/providers/agent/conf/";
/* URL to talk back to Agent Controller*/
String CONTROLLER_URL = "agent.controller.url";
/**
* The location of pre-installed agent path.
* This can be also be dynamically computed based on Yarn installation of agent.
*/
String PACKAGE_PATH = "agent.package.root";
/**
* The location of the script implementing the command.
*/
String SCRIPT_PATH = "agent.script";
/**
* Execution home for the agent.
*/
String APP_HOME = "app.home";
String APP_ROOT = "site.global.app_root";
String APP_CLIENT_ROOT = "client_root";
/**
* Runas user of the application
*/
String RUNAS_USER = "site.global.app_user";
/**
* Name of the service.
*/
String SERVICE_NAME = "app.name";
String ARG_LABEL = "--label";
String ARG_HOST = "--host";
String ARG_PORT = "--port";
String ARG_SECURED_PORT = "--secured_port";
String ARG_ZOOKEEPER_QUORUM = "--zk-quorum";
String ARG_ZOOKEEPER_REGISTRY_PATH = "--zk-reg-path";
String ARG_DEBUG = "--debug";
String AGENT_MAIN_SCRIPT_ROOT = "./infra/agent/slider-agent/";
String AGENT_JINJA2_ROOT = "./infra/agent/slider-agent/jinja2";
String AGENT_MAIN_SCRIPT = "agent/main.py";
String APP_DEF = "application.def";
String APP_DEF_ORIGINAL = "application.def.original";
String ADDON_PREFIX = "application.addon.";
String ADDONS = "application.addons";
String AGENT_VERSION = "agent.version";
String AGENT_CONF = "agent.conf";
String ADDON_FOR_ALL_COMPONENTS = "ALL";
String AGENT_INSTALL_DIR = "infra/agent";
String APP_DEFINITION_DIR = "app/definition";
String ADDON_DEFINITION_DIR = "addon/definition";
String AGENT_CONFIG_FILE = "infra/conf/agent.ini";
String AGENT_VERSION_FILE = "infra/version";
String PACKAGE_LIST = "package_list";
String WAIT_HEARTBEAT = "wait.heartbeat";
String PYTHON_EXE = "python";
String CREATE_DEF_ZK_NODE = "create.default.zookeeper.node";
String HEARTBEAT_MONITOR_INTERVAL = "heartbeat.monitor.interval";
String AGENT_INSTANCE_DEBUG_DATA = "agent.instance.debug.data";
String AGENT_OUT_FILE = "slider-agent.out";
String KEY_AGENT_TWO_WAY_SSL_ENABLED = "ssl.server.client.auth";
String INFRA_RUN_SECURITY_DIR = "infra/run/security/";
String CERT_FILE_LOCALIZATION_PATH = INFRA_RUN_SECURITY_DIR + "ca.crt";
String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
String TEST_RELAX_VERIFICATION = "test.relax.validation";
String DEFAULT_METAINFO_MAP_KEY = "DEFAULT_KEY";
}

38
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
View File

@ -17,37 +17,37 @@
*/
package org.apache.slider.providers.docker;
import org.apache.hadoop.conf.Configuration;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.slider.util.RestApiErrorMessages;
import java.util.Collections;
import java.util.List;
import java.io.IOException;
public class DockerClientProvider extends AbstractClientProvider
implements SliderKeys {
protected static final Logger log =
LoggerFactory.getLogger(DockerClientProvider.class);
private static final ProviderUtils providerUtils = new ProviderUtils(log);
protected static final String NAME = "docker";
public DockerClientProvider(Configuration conf) {
super(conf);
public DockerClientProvider() {
super();
}
@Override
public String getName() {
return NAME;
public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
if (artifact == null) {
throw new IllegalArgumentException(
RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
}
if (StringUtils.isEmpty(artifact.getId())) {
throw new IllegalArgumentException(
RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
}
}
@Override
public List<ProviderRole> getRoles() {
return Collections.emptyList();
protected void validateConfigFile(ConfigFile configFile, FileSystem
fileSystem) throws IOException {
}
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
View File

@ -27,7 +27,4 @@ public interface DockerKeys {
String DEFAULT_DOCKER_NETWORK = "bridge";
Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false;
String OUT_FILE = "stdout.txt";
String ERR_FILE = "stderr.txt";
}

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java
View File

@ -17,27 +17,36 @@
*/
package org.apache.slider.providers.docker;
import org.apache.hadoop.conf.Configuration;
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.SliderProviderFactory;
public class DockerProviderFactory extends SliderProviderFactory {
private static final SliderProviderFactory FACTORY = new
DockerProviderFactory();
public DockerProviderFactory() {
private DockerProviderFactory() {
}
public DockerProviderFactory(Configuration conf) {
super(conf);
private static class Client {
static final AbstractClientProvider PROVIDER = new DockerClientProvider();
}
private static class Server {
static final ProviderService PROVIDER = new DockerProviderService();
}
@Override
public AbstractClientProvider createClientProvider() {
return new DockerClientProvider(getConf());
return Client.PROVIDER;
}
@Override
public ProviderService createServerProvider() {
return new DockerProviderService();
return Server.PROVIDER;
}
public static SliderProviderFactory getInstance() {
return FACTORY;
}
}

140
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
View File

@ -17,159 +17,31 @@
*/
package org.apache.slider.providers.docker;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.Component;
import org.apache.slider.api.resource.ContainerState;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.CommandLineBuilder;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import org.apache.slider.providers.AbstractProviderService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Map;
import java.util.Map.Entry;
import static org.apache.slider.util.ServiceApiUtil.$;
public class DockerProviderService extends AbstractService
implements ProviderService, DockerKeys, SliderKeys {
public class DockerProviderService extends AbstractProviderService
implements DockerKeys {
protected static final Logger log =
LoggerFactory.getLogger(DockerProviderService.class);
private static final ProviderUtils providerUtils = new ProviderUtils(log);
private static final String QUICK_LINKS = "quicklinks";
protected StateAccessForProviders amState;
protected YarnRegistryViewForProviders yarnRegistry;
private ServiceTimelinePublisher serviceTimelinePublisher;
protected DockerProviderService() {
super("DockerProviderService");
super(DockerProviderService.class.getSimpleName());
}
@Override
public void setAMState(StateAccessForProviders stateAccessor) {
this.amState = stateAccessor;
}
@Override
public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) {
this.yarnRegistry = yarnRegistry;
}
public void buildContainerLaunchContext(ContainerLauncher launcher,
Application application, Container container, ProviderRole providerRole,
SliderFileSystem fileSystem, RoleInstance roleInstance)
throws IOException, SliderException {
Component component = providerRole.component;
public void processArtifact(ContainerLauncher launcher, Component
component, SliderFileSystem fileSystem) throws IOException {
launcher.setYarnDockerMode(true);
launcher.setDockerImage(component.getArtifact().getId());
launcher.setDockerNetwork(component.getConfiguration()
.getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK));
launcher.setRunPrivilegedContainer(component.getRunPrivilegedContainer());
// Generate tokens (key-value pair) for config substitution.
// Get pre-defined tokens
Map<String, String> globalTokens = amState.getGlobalSubstitutionTokens();
Map<String, String> tokensForSubstitution = providerUtils
.initCompTokensForSubstitute(roleInstance);
tokensForSubstitution.putAll(globalTokens);
// Set the environment variables in launcher
launcher.putEnv(SliderUtils
.buildEnvMap(component.getConfiguration(), tokensForSubstitution));
launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
if (System.getenv(HADOOP_USER_NAME) != null) {
launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
}
launcher.setEnv("LANG", "en_US.UTF-8");
launcher.setEnv("LC_ALL", "en_US.UTF-8");
launcher.setEnv("LANGUAGE", "en_US.UTF-8");
for (Entry<String, String> entry : launcher.getEnv().entrySet()) {
tokensForSubstitution.put($(entry.getKey()), entry.getValue());
}
providerUtils.addComponentHostTokens(tokensForSubstitution, amState);
// create config file on hdfs and add local resource
providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
component, tokensForSubstitution, roleInstance, amState);
// substitute launch command
String launchCommand = ProviderUtils
.substituteStrWithTokens(component.getLaunchCommand(),
tokensForSubstitution);
CommandLineBuilder operation = new CommandLineBuilder();
operation.add(launchCommand);
operation.addOutAndErrFiles(OUT_FILE, ERR_FILE);
launcher.addCommand(operation.build());
// publish exports
providerUtils
.substituteMapWithTokens(application.getQuicklinks(), tokensForSubstitution);
PublishedConfiguration pubconf = new PublishedConfiguration(QUICK_LINKS,
application.getQuicklinks().entrySet());
amState.getPublishedSliderConfigurations().put(QUICK_LINKS, pubconf);
if (serviceTimelinePublisher != null) {
serviceTimelinePublisher.serviceAttemptUpdated(application);
}
}
public boolean processContainerStatus(ContainerId containerId,
ContainerStatus status) {
log.debug("Handling container status: {}", status);
if (SliderUtils.isEmpty(status.getIPs()) ||
SliderUtils.isUnset(status.getHost())) {
return true;
}
RoleInstance instance = amState.getOwnedContainer(containerId);
if (instance == null) {
// container is completed?
return false;
}
try {
providerUtils.updateServiceRecord(amState, yarnRegistry,
containerId.toString(), instance.role, status.getIPs(), status.getHost());
} catch (IOException e) {
// could not write service record to ZK, log and retry
log.warn("Error updating container {} service record in registry, " +
"retrying", containerId, e);
return true;
}
// TODO publish ip and host
org.apache.slider.api.resource.Container container =
instance.providerRole.component.getContainer(containerId.toString());
if (container != null) {
container.setIp(StringUtils.join(",", status.getIPs()));
container.setHostname(status.getHost());
container.setState(ContainerState.READY);
} else {
log.warn(containerId + " not found in Application!");
}
return false;
}
@Override
public void setServiceTimelinePublisher(ServiceTimelinePublisher publisher) {
this.serviceTimelinePublisher = publisher;
}
}

65
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballClientProvider.java Normal file
View File

@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.tarball;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.util.RestApiErrorMessages;
import java.io.IOException;
import java.nio.file.Paths;
public class TarballClientProvider extends AbstractClientProvider
implements SliderKeys {
public TarballClientProvider() {
}
@Override
public void validateArtifact(Artifact artifact, FileSystem fs)
throws IOException {
if (artifact == null) {
throw new IllegalArgumentException(
RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
}
if (StringUtils.isEmpty(artifact.getId())) {
throw new IllegalArgumentException(
RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
}
Path p = new Path(artifact.getId());
if (!fs.exists(p)) {
throw new IllegalArgumentException( "Artifact tarball does not exist "
+ artifact.getId());
}
}
@Override
protected void validateConfigFile(ConfigFile configFile, FileSystem
fileSystem) throws IOException {
// validate dest_file is not absolute
if (Paths.get(configFile.getDestFile()).isAbsolute()) {
throw new IllegalArgumentException(
"Dest_file must not be absolute path: " + configFile.getDestFile());
}
}
}

52
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderFactory.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.tarball;
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.SliderProviderFactory;
public class TarballProviderFactory extends SliderProviderFactory {
private static final SliderProviderFactory FACTORY = new
TarballProviderFactory();
private TarballProviderFactory() {
}
private static class Client {
static final AbstractClientProvider PROVIDER = new TarballClientProvider();
}
private static class Server {
static final ProviderService PROVIDER = new TarballProviderService();
}
@Override
public AbstractClientProvider createClientProvider() {
return Client.PROVIDER;
}
@Override
public ProviderService createServerProvider() {
return Server.PROVIDER;
}
public static SliderProviderFactory getInstance() {
return FACTORY;
}
}

50
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/tarball/TarballProviderService.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.tarball;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.slider.api.resource.Component;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.providers.AbstractProviderService;
import java.io.IOException;
public class TarballProviderService extends AbstractProviderService {
protected TarballProviderService() {
super(TarballProviderService.class.getSimpleName());
}
@Override
public void processArtifact(ContainerLauncher launcher, Component
component, SliderFileSystem fileSystem) throws IOException {
Path artifact = new Path(component.getArtifact().getId());
if (!fileSystem.isFile(artifact)) {
throw new IOException("Package doesn't exist as a resource: " +
artifact.toString());
}
log.info("Adding resource {}", artifact.toString());
LocalResourceType type = LocalResourceType.ARCHIVE;
LocalResource packageResource = fileSystem.createAmResource(
artifact, type);
launcher.addLocalResource(APP_INSTALL_DIR, packageResource);
}
}

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
View File

@ -27,7 +27,7 @@ import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.providers.SliderProviderFactory;
import org.apache.slider.server.appmaster.actions.ActionStartContainer;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.ContainerAssignment;
@ -43,6 +43,8 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.apache.slider.common.SliderKeys.KEY_CONTAINER_LAUNCH_DELAY;
/**
* A service for launching containers
*/
@ -59,11 +61,6 @@ public class RoleLaunchService
*/
private final QueueAccess actionQueue;
/**
* Provider building up the command
*/
private final ProviderService provider;
/**
* Filesystem to use for the launch
*/
@ -75,16 +72,14 @@ public class RoleLaunchService
/**
* Construct an instance of the launcher
* @param queueAccess
* @param provider the provider
* @param fs filesystem
* @param envVars environment variables
*/
public RoleLaunchService(QueueAccess queueAccess, ProviderService provider,
SliderFileSystem fs, Map<String, String> envVars) {
public RoleLaunchService(QueueAccess queueAccess, SliderFileSystem fs,
Map<String, String> envVars) {
super(ROLE_LAUNCH_SERVICE);
this.actionQueue = queueAccess;
this.fs = fs;
this.provider = provider;
this.envVars = envVars;
}
@ -167,11 +162,13 @@ public class RoleLaunchService
instance.roleId = role.id;
instance.environment = envDescription;
ProviderService provider = SliderProviderFactory.getProviderService(
role.component.getArtifact());
provider.buildContainerLaunchContext(containerLauncher, application,
container, role, fs, instance);
long delay = role.component.getConfiguration()
.getPropertyLong(AgentKeys.KEY_CONTAINER_LAUNCH_DELAY, 0);
.getPropertyLong(KEY_CONTAINER_LAUNCH_DELAY, 0);
long maxDelay = getConfig()
.getLong(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS);

64
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
View File

@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.TimelineV2Client;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
@ -342,7 +341,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* ProviderService of this cluster
*/
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private ProviderService providerService;
private List<ProviderService> providers = new ArrayList<>();
/**
* The YARN registry service
@ -523,8 +522,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public Configuration bindArgs(Configuration config, String... args) throws Exception {
// let the superclass process it
Configuration superConf = super.bindArgs(config, args);
// add the slider XML config
ConfigHelper.injectSliderXMLResource();
//yarn-ify
YarnConfiguration yarnConfiguration = new YarnConfiguration(
@ -603,12 +600,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// obtain security state
// set the global security flag for the instance definition
//get our provider
SliderProviderFactory factory =
SliderProviderFactory.createSliderProviderFactory("docker");
providerService = factory.createServerProvider();
// init the provider BUT DO NOT START IT YET
initAndAddService(providerService);
// initialize our providers
for (Component component : application.getComponents()) {
SliderProviderFactory factory = SliderProviderFactory
.createSliderProviderFactory(component.getArtifact());
ProviderService providerService = factory.createServerProvider();
// init the provider BUT DO NOT START IT YET
initAndAddService(providerService);
providers.add(providerService);
}
InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf);
log.info("RM is at {}", rmSchedulerAddress);
@ -667,7 +667,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
serviceTimelinePublisher.init(getConfig());
serviceTimelinePublisher.start();
providerService.setServiceTimelinePublisher(serviceTimelinePublisher);
for (ProviderService providerService : providers) {
providerService.setServiceTimelinePublisher(serviceTimelinePublisher);
}
appState.setServiceTimelinePublisher(serviceTimelinePublisher);
log.info("ServiceTimelinePublisher started.");
}
@ -707,7 +709,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
WebAppApiImpl webAppApi =
new WebAppApiImpl(
stateForProviders,
providerService, registryOperations,
registryOperations,
metricsAndMonitoring,
actionQueues);
initAMFilterOptions(serviceConf);
@ -843,13 +845,14 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//launcher service
launchService = new RoleLaunchService(actionQueues,
providerService,
fs, envVars);
deployChildService(launchService);
//Give the provider access to the state, and AM
providerService.setAMState(stateForProviders);
for (ProviderService providerService : providers) {
providerService.setAMState(stateForProviders);
}
// chaos monkey
maybeStartMonkey();
@ -1119,7 +1122,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
SliderKeys.APP_TYPE,
instanceName,
appAttemptID);
providerService.bindToYarnRegistry(yarnRegistryOperations);
for (ProviderService providerService : providers) {
providerService.bindToYarnRegistry(yarnRegistryOperations);
}
// Yarn registry
ServiceRecord serviceRecord = new ServiceRecord();
@ -1859,7 +1864,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
protected synchronized void launchProviderService()
throws IOException, SliderException {
// didn't start, so don't register
providerService.start();
for (ProviderService providerService : providers) {
providerService.start();
}
// and send the started event ourselves
eventCallbackEvent(null);
}
@ -1959,19 +1966,23 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
ContainerStatus containerStatus) {
LOG_YARN.debug("Container Status: id={}, status={}", containerId,
containerStatus);
RoleInstance cinfo = appState.getOwnedContainer(containerId);
if (cinfo == null) {
LOG_YARN.error("Owned container not found for {}", containerId);
return;
}
ProviderService providerService = SliderProviderFactory
.getProviderService(cinfo.providerRole.component.getArtifact());
if (providerService.processContainerStatus(containerId, containerStatus)) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
}
RoleInstance cinfo = appState.getOwnedContainer(containerId);
if (cinfo != null) {
LOG_YARN.info("Re-requesting status for role {}, {}",
cinfo.role, containerId);
//trigger another async container status
nmClientAsync.getContainerStatusAsync(containerId,
cinfo.container.getNodeId());
}
LOG_YARN.info("Re-requesting status for role {}, {}",
cinfo.role, containerId);
//trigger another async container status
nmClientAsync.getContainerStatusAsync(containerId,
cinfo.container.getNodeId());
} else if (timelineServiceEnabled) {
RoleInstance instance = appState.getOwnedContainer(containerId);
if (instance != null) {
@ -1997,11 +2008,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
LOG_YARN.warn("Failed to stop Container {}", containerId);
}
public ProviderService getProviderService() {
return providerService;
}
/**
* Queue an action for immediate execution in the executor thread
* @param action action to execute

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
View File

@ -17,7 +17,6 @@
package org.apache.slider.server.appmaster.web;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.state.AppState;
@ -33,11 +32,6 @@ public interface WebAppApi {
*/
StateAccessForProviders getAppState();
/**
* The {@link ProviderService} for the current cluster
*/
ProviderService getProviderService();
/**
* Registry operations accessor
* @return registry access

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
View File

@ -17,7 +17,6 @@
package org.apache.slider.server.appmaster.web;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
@ -33,21 +32,18 @@ public class WebAppApiImpl implements WebAppApi {
private static final Logger log = LoggerFactory.getLogger(WebAppApiImpl.class);
protected final StateAccessForProviders appState;
protected final ProviderService provider;
private final RegistryOperations registryOperations;
private final MetricsAndMonitoring metricsAndMonitoring;
private final QueueAccess queues;
public WebAppApiImpl(StateAccessForProviders appState,
ProviderService provider, RegistryOperations registryOperations,
RegistryOperations registryOperations,
MetricsAndMonitoring metricsAndMonitoring, QueueAccess queues) {
checkNotNull(appState);
checkNotNull(provider);
this.queues = queues;
this.registryOperations = registryOperations;
this.appState = appState;
this.provider = provider;
this.metricsAndMonitoring = metricsAndMonitoring;
}
@ -56,11 +52,6 @@ public class WebAppApiImpl implements WebAppApi {
return appState;
}
@Override
public ProviderService getProviderService() {
return provider;
}
@Override
public RegistryOperations getRegistryOperations() {
return registryOperations;

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java
View File

@ -19,7 +19,6 @@
package org.apache.slider.server.appmaster.web.view;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
@ -33,12 +32,10 @@ import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_
public abstract class SliderHamletBlock extends HtmlBlock {
protected final StateAccessForProviders appState;
protected final ProviderService providerService;
protected final RestPaths restPaths = new RestPaths();
public SliderHamletBlock(WebAppApi slider) {
this.appState = slider.getAppState();
this.providerService = slider.getProviderService();
}
protected String rootPath(String absolutePath) {

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/slider.xml
View File

@ -1,30 +0,0 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<configuration>
<property>
<name>slider.config.loaded</name>
<value>true</value>
</property>
<property>
<name>slider.provider.docker</name>
<value>org.apache.slider.providers.docker.DockerProviderFactory</value>
</property>
</configuration>

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java
View File

@ -18,11 +18,16 @@
package org.apache.slider.providers;
import org.apache.slider.providers.docker.DockerKeys;
import org.apache.slider.api.resource.Artifact;
import org.apache.slider.api.resource.Artifact.TypeEnum;
import org.apache.slider.providers.docker.DockerClientProvider;
import org.apache.slider.providers.docker.DockerProviderFactory;
import org.apache.slider.providers.docker.DockerProviderService;
import org.apache.slider.providers.tarball.TarballClientProvider;
import org.apache.slider.providers.tarball.TarballProviderFactory;
import org.apache.slider.providers.tarball.TarballProviderService;
import org.junit.Test;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
/**
@ -30,25 +35,38 @@ import static org.junit.Assert.assertTrue;
*/
public class TestProviderFactory {
@Test
public void testLoadAgentProvider() throws Throwable {
public void testDockerFactory() throws Throwable {
SliderProviderFactory factory = SliderProviderFactory
.createSliderProviderFactory(DockerKeys.PROVIDER_DOCKER);
.createSliderProviderFactory(new Artifact().type(TypeEnum.DOCKER));
assertTrue(factory instanceof DockerProviderFactory);
assertTrue(factory.createClientProvider() instanceof DockerClientProvider);
assertTrue(factory.createServerProvider() instanceof DockerProviderService);
assertTrue(SliderProviderFactory.getProviderService(new Artifact()
.type(TypeEnum.DOCKER)) instanceof DockerProviderService);
}
@Test
public void testCreateClientProvider() throws Throwable {
public void testTarballFactory() throws Throwable {
SliderProviderFactory factory = SliderProviderFactory
.createSliderProviderFactory(DockerKeys.PROVIDER_DOCKER);
assertNotNull(factory.createClientProvider());
.createSliderProviderFactory(new Artifact().type(TypeEnum.TARBALL));
assertTrue(factory instanceof TarballProviderFactory);
assertTrue(factory.createClientProvider() instanceof TarballClientProvider);
assertTrue(factory.createServerProvider() instanceof
TarballProviderService);
assertTrue(SliderProviderFactory.getProviderService(new Artifact()
.type(TypeEnum.TARBALL)) instanceof TarballProviderService);
}
@Test
public void testCreateProviderByClassname() throws Throwable {
public void testDefaultFactory() throws Throwable {
SliderProviderFactory factory = SliderProviderFactory
.createSliderProviderFactory(DockerKeys.PROVIDER_DOCKER);
assertNotNull(factory.createServerProvider());
assertTrue(factory instanceof DockerProviderFactory);
.createSliderProviderFactory(null);
assertTrue(factory instanceof DefaultProviderFactory);
assertTrue(factory.createClientProvider() instanceof DefaultClientProvider);
assertTrue(factory.createServerProvider() instanceof
DefaultProviderService);
assertTrue(SliderProviderFactory.getProviderService(null) instanceof
DefaultProviderService);
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.java
View File

@ -20,9 +20,7 @@ import com.google.inject.AbstractModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
import org.apache.slider.server.appmaster.model.mock.MockProviderService;
import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.WebAppApiImpl;
@ -45,11 +43,9 @@ public class TestClusterSpecificationBlock extends BaseMockAppStateTest {
ProviderAppState providerAppState = new ProviderAppState(
"undefined",
appState);
ProviderService providerService = new MockProviderService();
WebAppApiImpl inst = new WebAppApiImpl(
providerAppState,
providerService,
null,
null, null);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestContainerStatsBlock.java
View File

@ -27,12 +27,10 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
import org.apache.hadoop.yarn.webapp.hamlet.HamletImpl.EImp;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
import org.apache.slider.server.appmaster.model.mock.MockContainer;
import org.apache.slider.server.appmaster.model.mock.MockContainerId;
import org.apache.slider.server.appmaster.model.mock.MockNodeId;
import org.apache.slider.server.appmaster.model.mock.MockProviderService;
import org.apache.slider.server.appmaster.model.mock.MockResource;
import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.state.RoleInstance;
@ -65,14 +63,12 @@ public class TestContainerStatsBlock extends BaseMockAppStateTest {
@Before
public void setup() throws Exception {
super.setup();
ProviderService providerService = new MockProviderService();
ProviderAppState providerAppState = new ProviderAppState(
"undefined",
appState);
WebAppApiImpl inst = new WebAppApiImpl(
providerAppState,
providerService,
null,
METRICS, null);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/view/TestIndexBlock.java
View File

@ -22,12 +22,10 @@ import com.google.inject.Injector;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.model.appstate.BaseMockAppStateAATest;
import org.apache.slider.server.appmaster.model.mock.MockContainer;
import org.apache.slider.server.appmaster.model.mock.MockContainerId;
import org.apache.slider.server.appmaster.model.mock.MockNodeId;
import org.apache.slider.server.appmaster.model.mock.MockProviderService;
import org.apache.slider.server.appmaster.model.mock.MockResource;
import org.apache.slider.server.appmaster.state.ContainerOutcome;
import org.apache.slider.server.appmaster.state.OutstandingRequest;
@ -58,14 +56,12 @@ public class TestIndexBlock extends BaseMockAppStateAATest {
public void setup() throws Exception {
super.setup();
assertNotNull(appState);
ProviderService providerService = new MockProviderService();
ProviderAppState providerAppState = new ProviderAppState(
"undefined",
appState);
WebAppApiImpl inst = new WebAppApiImpl(
providerAppState,
providerService,
null,
METRICS, null);