Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-5909. Remove agent related code in slider AM. Contributed by Jian He

This commit is contained in:
Jian He 2016-11-29 13:06:45 -08:00
parent 2c4e0f33e9
commit 234dba84a1
83 changed files with 26 additions and 11517 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-project/pom.xml
View File

@ -965,11 +965,6 @@
<artifactId>jackson-module-jaxb-annotations</artifactId>
<version>${jackson2.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-json-provider</artifactId>
<version>${jackson2.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-cbor</artifactId>

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
View File

@ -30,7 +30,7 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.services.api.impl.ApplicationApiService;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.mortbay.jetty.webapp.Configuration;
import org.eclipse.jetty.webapp.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
View File

@ -225,7 +225,7 @@
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
<artifactId>javax.servlet-api</artifactId>
</dependency>
<dependency>
@ -288,21 +288,6 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-sslengine</artifactId>
</dependency>
<dependency>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
@ -314,12 +299,6 @@
<artifactId>jettison</artifactId>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-sslengine</artifactId>
<version>6.1.26</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
View File

@ -70,7 +70,6 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ClasspathConstructor;
import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.server.services.utility.PatternValidator;
import org.apache.slider.server.services.workflow.ForkedProcessService;
import org.apache.zookeeper.server.util.KerberosUtil;

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
View File

@ -19,17 +19,17 @@
package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
import org.apache.hadoop.registry.client.types.AddressTypes;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
import org.apache.hadoop.registry.client.types.AddressTypes;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
@ -44,7 +44,6 @@ import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.services.workflow.ForkedProcessService;
import org.apache.slider.server.services.workflow.ServiceParent;
import org.apache.slider.server.services.workflow.WorkflowSequenceService;
@ -77,7 +76,6 @@ public abstract class AbstractProviderService
private static final Logger log =
LoggerFactory.getLogger(AbstractProviderService.class);
protected StateAccessForProviders amState;
protected AgentRestOperations restOps;
protected URL amWebAPI;
protected YarnRegistryViewForProviders yarnRegistry;
protected QueueAccess queueAccess;
@ -126,19 +124,10 @@ public abstract class AbstractProviderService
return yarnRegistry;
}
@Override
public AgentRestOperations getAgentRestOperations() {
return restOps;
}
@Override
public void notifyContainerCompleted(ContainerId containerId) {
}
public void setAgentRestOperations(AgentRestOperations agentRestOperations) {
this.restOps = agentRestOperations;
}
/**
* Load default Configuration
* @param confDir configuration directory

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
View File

@ -20,10 +20,10 @@ package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.tools.SliderFileSystem;
@ -37,7 +37,6 @@ import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import java.io.File;
@ -175,12 +174,6 @@ public interface ProviderService extends ProviderCore,
*/
void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry);
/**
* Returns the agent rest operations interface.
* @return the interface if available, null otherwise.
*/
AgentRestOperations getAgentRestOperations();
/**
* Build up the endpoint details for this service
* @param details

713
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
View File

@ -1,713 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
import com.google.common.io.Files;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.client.ClientUtils;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.AbstractLauncher;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.providers.agent.application.metadata.Application;
import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.providers.agent.application.metadata.ConfigFile;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
import org.apache.slider.providers.agent.application.metadata.OSPackage;
import org.apache.slider.providers.agent.application.metadata.OSSpecific;
import org.apache.slider.providers.agent.application.metadata.Package;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.HashMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import static org.apache.slider.common.tools.SliderUtils.getApplicationDefinitionPath;
/** This class implements the client-side aspects of the agent deployer */
public class AgentClientProvider extends AbstractClientProvider
implements AgentKeys, SliderKeys {
protected static final Logger log =
LoggerFactory.getLogger(AgentClientProvider.class);
protected static final String NAME = "agent";
private static final ProviderUtils providerUtils = new ProviderUtils(log);
public static final String E_COULD_NOT_READ_METAINFO
= "Not a valid app package. Could not read metainfo.";
protected Map<String, Metainfo> metaInfoMap = new ConcurrentHashMap<String, Metainfo>();
protected AgentClientProvider(Configuration conf) {
super(conf);
}
@Override
public String getName() {
return NAME;
}
@Override
public List<ProviderRole> getRoles() {
return AgentRoles.getRoles();
}
@Override //Client
public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem,
String clustername,
Configuration configuration,
AggregateConf instanceDefinition,
Path clusterDirPath,
Path generatedConfDirPath,
boolean secure) throws
SliderException,
IOException {
super.preflightValidateClusterConfiguration(sliderFileSystem, clustername,
configuration,
instanceDefinition,
clusterDirPath,
generatedConfDirPath, secure);
String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
.getAppConfOperations());
Path appDefPath = new Path(appDef);
sliderFileSystem.verifyFileExists(appDefPath);
String agentConf = instanceDefinition.getAppConfOperations().
getGlobalOptions().getOption(AGENT_CONF, "");
if (StringUtils.isNotEmpty(agentConf)) {
sliderFileSystem.verifyFileExists(new Path(agentConf));
}
String appHome = instanceDefinition.getAppConfOperations().
getGlobalOptions().get(PACKAGE_PATH);
if (SliderUtils.isUnset(appHome)) {
String agentImage = instanceDefinition.getInternalOperations().
get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
sliderFileSystem.verifyFileExists(new Path(agentImage));
}
}
@Override
public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws
SliderException {
super.validateInstanceDefinition(instanceDefinition, fs);
log.debug("Validating conf {}", instanceDefinition);
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
providerUtils.validateNodeCount(instanceDefinition, ROLE_NODE,
0, -1);
String appDef = null;
try {
// Validate the app definition
appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
.getAppConfOperations());
} catch (BadConfigException bce) {
throw new BadConfigException("Application definition must be provided. " + bce.getMessage());
}
log.info("Validating app definition {}", appDef);
String extension = appDef.substring(appDef.lastIndexOf(".") + 1, appDef.length());
if (!"zip".equals(extension.toLowerCase(Locale.ENGLISH))) {
throw new BadConfigException("App definition must be packaged as a .zip file. File provided is " + appDef);
}
Set<String> names = resources.getComponentNames();
names.remove(COMPONENT_AM);
Map<Integer, String> priorityMap = new HashMap<Integer, String>();
for (String name : names) {
try {
// Validate the app definition
appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
.getAppConfOperations(), name);
} catch (BadConfigException bce) {
throw new BadConfigException("Application definition must be provided. " + bce.getMessage());
}
Metainfo metaInfo = getMetainfo(fs, appDef);
MapOperations component = resources.getMandatoryComponent(name);
if (metaInfo != null) {
Component componentDef = metaInfo.getApplicationComponent(
AgentUtils.getMetainfoComponentName(name,
instanceDefinition.getAppConfOperations()));
if (componentDef == null) {
throw new BadConfigException(
"Component %s is not a member of application.", name);
}
}
int priority =
component.getMandatoryOptionInt(ResourceKeys.COMPONENT_PRIORITY);
if (priority <= 0) {
throw new BadConfigException("Component %s %s value out of range %d",
name,
ResourceKeys.COMPONENT_PRIORITY,
priority);
}
String existing = priorityMap.get(priority);
if (existing != null) {
throw new BadConfigException(
"Component %s has a %s value %d which duplicates that of %s",
name,
ResourceKeys.COMPONENT_PRIORITY,
priority,
existing);
}
priorityMap.put(priority, name);
// fileSystem may be null for tests
if (metaInfo != null) {
Component componentDef = metaInfo.getApplicationComponent(
AgentUtils.getMetainfoComponentName(name,
instanceDefinition.getAppConfOperations()));
// ensure that intance count is 0 for client components
if ("CLIENT".equals(componentDef.getCategory())) {
MapOperations componentConfig = resources.getMandatoryComponent(name);
int count =
componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
if (count > 0) {
throw new BadConfigException("Component %s is of type CLIENT and cannot be instantiated."
+ " Use \"slider client install ...\" command instead.",
name);
}
} else {
MapOperations componentConfig = resources.getMandatoryComponent(name);
int count =
componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
int definedMinCount = componentDef.getMinInstanceCountInt();
int definedMaxCount = componentDef.getMaxInstanceCountInt();
if (count < definedMinCount || count > definedMaxCount) {
throw new BadConfigException("Component %s, %s value %d out of range. "
+ "Expected minimum is %d and maximum is %d",
name,
ResourceKeys.COMPONENT_INSTANCES,
count,
definedMinCount,
definedMaxCount);
}
}
}
}
}
@Override
public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem,
Configuration serviceConf,
AbstractLauncher launcher,
AggregateConf instanceDefinition,
Path snapshotConfDirPath,
Path generatedConfDirPath,
Configuration clientConfExtras,
String libdir,
Path tempPath,
boolean miniClusterTestRun) throws
IOException,
SliderException {
String agentImage = instanceDefinition.getInternalOperations().
get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
if (SliderUtils.isUnset(agentImage)) {
Path agentPath = new Path(tempPath.getParent(), PROVIDER_AGENT);
log.info("Automatically uploading the agent tarball at {}", agentPath);
fileSystem.getFileSystem().mkdirs(agentPath);
if (ProviderUtils.addAgentTar(this, AGENT_TAR, fileSystem, agentPath)) {
instanceDefinition.getInternalOperations().set(
InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH,
new Path(agentPath, AGENT_TAR).toUri());
}
}
}
@Override
public Set<String> getApplicationTags(SliderFileSystem fileSystem,
ConfTreeOperations appConf) throws SliderException {
return getApplicationTags(fileSystem,
getApplicationDefinitionPath(appConf));
}
public Set<String> getApplicationTags(SliderFileSystem fileSystem,
String appDef) throws SliderException {
Set<String> tags;
Metainfo metaInfo = getMetainfo(fileSystem, appDef);
if (metaInfo == null) {
log.error("Error retrieving metainfo from {}", appDef);
throw new SliderException("Error parsing metainfo file, possibly bad structure.");
}
Application application = metaInfo.getApplication();
tags = new HashSet<String>();
tags.add("Name: " + application.getName());
tags.add("Version: " + application.getVersion());
tags.add("Description: " + SliderUtils.truncate(application.getComment(), 80));
return tags;
}
@Override
public void processClientOperation(SliderFileSystem fileSystem,
RegistryOperations rops,
Configuration configuration,
String operation,
File clientInstallPath,
File appPackage,
JSONObject config,
String name) throws SliderException {
// create temp folder
// create sub-folders app_pkg, agent_pkg, command
File tmpDir = Files.createTempDir();
log.info("Command is being executed at {}", tmpDir.getAbsolutePath());
File appPkgDir = new File(tmpDir, "app_pkg");
appPkgDir.mkdir();
File agentPkgDir = new File(tmpDir, "agent_pkg");
agentPkgDir.mkdir();
File cmdDir = new File(tmpDir, "command");
cmdDir.mkdir();
Metainfo metaInfo = null;
JSONObject defaultConfig = null;
try {
// expand app package into /app_pkg
ZipInputStream zipInputStream = null;
try {
zipInputStream = new ZipInputStream(new FileInputStream(appPackage));
{
ZipEntry zipEntry = zipInputStream.getNextEntry();
while (zipEntry != null) {
log.info("Processing {}", zipEntry.getName());
String filePath = appPkgDir + File.separator + zipEntry.getName();
if (!zipEntry.isDirectory()) {
log.info("Extracting file {}", filePath);
extractFile(zipInputStream, filePath);
if ("metainfo.xml".equals(zipEntry.getName())) {
FileInputStream input = null;
try {
input = new FileInputStream(filePath);
metaInfo = new MetainfoParser().fromXmlStream(input);
} finally {
IOUtils.closeStream(input);
}
} else if ("metainfo.json".equals(zipEntry.getName())) {
FileInputStream input = null;
try {
input = new FileInputStream(filePath);
metaInfo = new MetainfoParser().fromJsonStream(input);
} finally {
IOUtils.closeStream(input);
}
} else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) {
try {
defaultConfig = new JSONObject(FileUtils.readFileToString(new File(filePath), Charset.defaultCharset()));
} catch (JSONException jex) {
throw new SliderException("Unable to read default client config.", jex);
}
}
} else {
log.info("Creating dir {}", filePath);
File dir = new File(filePath);
dir.mkdir();
}
zipInputStream.closeEntry();
zipEntry = zipInputStream.getNextEntry();
}
}
} finally {
zipInputStream.close();
}
if (metaInfo == null) {
throw new BadConfigException(E_COULD_NOT_READ_METAINFO);
}
String clientScript = null;
String clientComponent = null;
for (Component component : metaInfo.getApplication().getComponents()) {
if (component.getCategory().equals("CLIENT")) {
clientComponent = component.getName();
if (component.getCommandScript() != null) {
clientScript = component.getCommandScript().getScript();
}
break;
}
}
if (SliderUtils.isUnset(clientScript)) {
log.info("Installing CLIENT without script");
List<Package> packages = metaInfo.getApplication().getPackages();
if (packages.size() > 0) {
// retrieve package resources from HDFS and extract
for (Package pkg : packages) {
Path pkgPath = fileSystem.buildResourcePath(pkg.getName());
if (!fileSystem.isFile(pkgPath) && name != null) {
pkgPath = fileSystem.buildResourcePath(name, pkg.getName());
}
if (!fileSystem.isFile(pkgPath)) {
throw new IOException("Package doesn't exist as a resource: " +
pkg.getName());
}
if ("archive".equals(pkg.getType())) {
File pkgFile = new File(tmpDir, pkg.getName());
fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile);
expandTar(pkgFile, clientInstallPath);
} else {
File pkgFile = new File(clientInstallPath, pkg.getName());
fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile);
}
}
} else {
// extract tarball from app def
for (OSSpecific osSpecific : metaInfo.getApplication()
.getOSSpecifics()) {
for (OSPackage pkg : osSpecific.getPackages()) {
if ("tarball".equals(pkg.getType())) {
File pkgFile = new File(appPkgDir, pkg.getName());
expandTar(pkgFile, clientInstallPath);
}
}
}
}
if (name == null) {
log.warn("Conf files not being generated because no app name was " +
"provided");
return;
}
File confInstallDir;
String clientRoot = null;
if (config != null) {
try {
clientRoot = config.getJSONObject("global")
.getString(APP_CLIENT_ROOT);
} catch (JSONException e) {
log.info("Couldn't read {} from provided client config, falling " +
"back on default", APP_CLIENT_ROOT);
}
}
if (clientRoot == null && defaultConfig != null) {
try {
clientRoot = defaultConfig.getJSONObject("global")
.getString(APP_CLIENT_ROOT);
} catch (JSONException e) {
log.info("Couldn't read {} from default client config, using {}",
APP_CLIENT_ROOT, clientInstallPath);
}
}
if (clientRoot == null) {
confInstallDir = clientInstallPath;
} else {
confInstallDir = new File(new File(clientInstallPath, clientRoot), "conf");
if (!confInstallDir.exists()) {
confInstallDir.mkdirs();
}
}
String user = RegistryUtils.currentUser();
for (ConfigFile configFile : metaInfo.getComponentConfigFiles(clientComponent)) {
retrieveConfigFile(rops, configuration, configFile, name, user,
confInstallDir);
}
} else {
log.info("Installing CLIENT using script {}", clientScript);
expandAgentTar(agentPkgDir);
JSONObject commandJson = getCommandJson(defaultConfig, config, metaInfo, clientInstallPath, name);
FileWriter file = new FileWriter(new File(cmdDir, "command.json"));
try {
file.write(commandJson.toString());
} catch (IOException e) {
log.error("Couldn't write command.json to file");
} finally {
file.flush();
file.close();
}
runCommand(appPkgDir, agentPkgDir, cmdDir, clientScript);
}
} catch (IOException ioex) {
log.warn("Error while executing INSTALL command {}", ioex.getMessage());
throw new SliderException("INSTALL client failed.");
}
}
protected void runCommand(
File appPkgDir,
File agentPkgDir,
File cmdDir,
String clientScript) throws SliderException {
int exitCode = 0;
Exception exp = null;
try {
String clientScriptPath = appPkgDir.getAbsolutePath() + File.separator + "package" +
File.separator + clientScript;
List<String> command = Arrays.asList(PYTHON_EXE,
"-S",
clientScriptPath,
"INSTALL",
cmdDir.getAbsolutePath() + File.separator + "command.json",
appPkgDir.getAbsolutePath() + File.separator + "package",
cmdDir.getAbsolutePath() + File.separator + "command-out.json",
"DEBUG");
ProcessBuilder pb = new ProcessBuilder(command);
log.info("Command: " + StringUtils.join(pb.command(), " "));
pb.environment().put(PYTHONPATH,
agentPkgDir.getAbsolutePath()
+ File.separator + "slider-agent" + File.pathSeparator
+ agentPkgDir.getAbsolutePath()
+ File.separator + "slider-agent/jinja2");
log.info("{}={}", PYTHONPATH, pb.environment().get(PYTHONPATH));
Process proc = pb.start();
InputStream stderr = proc.getErrorStream();
InputStream stdout = proc.getInputStream();
BufferedReader stdOutReader = new BufferedReader(new InputStreamReader(stdout));
BufferedReader stdErrReader = new BufferedReader(new InputStreamReader(stderr));
proc.waitFor();
String line;
while ((line = stdOutReader.readLine()) != null) {
log.info("Stdout: " + line);
}
while ((line = stdErrReader.readLine()) != null) {
log.info("Stderr: " + line);
}
exitCode = proc.exitValue();
log.info("Exit value is {}", exitCode);
} catch (IOException e) {
exp = e;
} catch (InterruptedException e) {
exp = e;
}
if (exitCode != 0) {
throw new SliderException("INSTALL client failed with exit code " + exitCode);
}
if (exp != null) {
log.error("Error while executing INSTALL command {}. Stack trace {}",
exp.getMessage(),
ExceptionUtils.getStackTrace(exp));
throw new SliderException("INSTALL client failed.", exp);
}
}
private void expandAgentTar(File agentPkgDir) throws IOException {
String libDirProp = SliderUtils.getLibDir();
File tarFile = new File(libDirProp, AGENT_TAR);
expandTar(tarFile, agentPkgDir);
}
private void expandTar(File tarFile, File destDir) throws IOException {
log.info("Expanding tar {} to {}", tarFile, destDir);
TarArchiveInputStream tarIn = new TarArchiveInputStream(
new GzipCompressorInputStream(
new BufferedInputStream(
new FileInputStream(tarFile)
)
)
);
try {
TarArchiveEntry tarEntry = tarIn.getNextTarEntry();
while (tarEntry != null) {
File destPath = new File(destDir, tarEntry.getName());
File parent = destPath.getParentFile();
if (!parent.exists()) {
parent.mkdirs();
}
if (tarEntry.isDirectory()) {
destPath.mkdirs();
} else {
byte[] byteToRead = new byte[1024];
BufferedOutputStream buffOut =
new BufferedOutputStream(new FileOutputStream(destPath));
try {
int len;
while ((len = tarIn.read(byteToRead)) != -1) {
buffOut.write(byteToRead, 0, len);
}
} finally {
buffOut.close();
}
}
if ((tarEntry.getMode() & 0100) != 0) {
destPath.setExecutable(true);
}
tarEntry = tarIn.getNextTarEntry();
}
} finally {
tarIn.close();
}
}
private void retrieveConfigFile(RegistryOperations rops,
Configuration configuration, ConfigFile configFile, String name,
String user, File destDir) throws IOException, SliderException {
log.info("Retrieving config {} to {}", configFile.getDictionaryName(),
destDir);
PublishedConfiguration published = ClientUtils.getConfigFromRegistry(rops,
configuration, configFile.getDictionaryName(), name, user, true);
ClientUtils.saveOrReturnConfig(published, configFile.getType(),
destDir, configFile.getFileName());
}
protected JSONObject getCommandJson(JSONObject defaultConfig,
JSONObject inputConfig,
Metainfo metainfo,
File clientInstallPath,
String name) throws SliderException {
try {
JSONObject pkgList = new JSONObject();
pkgList.put(PACKAGE_LIST,
AgentProviderService.getPackageListFromApplication(metainfo.getApplication()));
JSONObject obj = new JSONObject();
obj.put("hostLevelParams", pkgList);
String user = RegistryUtils.currentUser();
JSONObject configuration = new JSONObject();
JSONObject global = new JSONObject();
global.put("app_install_dir", clientInstallPath.getAbsolutePath());
global.put("app_user", user);
if (name != null) {
global.put("app_name", name);
}
if (defaultConfig != null) {
readConfigEntries(defaultConfig, clientInstallPath, global, name, user);
}
if (inputConfig != null) {
readConfigEntries(inputConfig, clientInstallPath, global, name, user);
}
configuration.put("global", global);
obj.put("configurations", configuration);
return obj;
} catch (JSONException jex) {
log.warn("Error while executing INSTALL command {}", jex.getMessage());
throw new SliderException("INSTALL client failed.");
}
}
private void readConfigEntries(JSONObject inpConfig,
File clientInstallPath,
JSONObject globalConfig,
String name, String user)
throws JSONException {
JSONObject globalSection = inpConfig.getJSONObject("global");
Iterator it = globalSection.keys();
while (it.hasNext()) {
String key = (String) it.next();
String value = globalSection.getString(key);
if (SliderUtils.isSet(value)) {
value = value.replace("{app_install_dir}", clientInstallPath.getAbsolutePath());
value = value.replace("{app_user}", user);
if (name != null) {
value = value.replace("{app_name}", name);
}
}
if (globalConfig.has(key)) {
// last one wins
globalConfig.remove(key);
}
globalConfig.put(key, value);
}
}
private void extractFile(ZipInputStream zipInputStream, String filePath) throws IOException {
BufferedOutputStream output = new BufferedOutputStream(new FileOutputStream(filePath));
try {
byte[] bytesRead = new byte[4096];
int read = 0;
while ((read = zipInputStream.read(bytesRead)) != -1) {
output.write(bytesRead, 0, read);
}
} finally {
output.close();
}
}
private Metainfo getMetainfo(SliderFileSystem fs, String appDef) {
Metainfo metaInfo = metaInfoMap.get(appDef);
if (fs != null && metaInfo == null) {
try {
metaInfo = AgentUtils.getApplicationMetainfo(fs, appDef, false);
metaInfoMap.put(appDef, metaInfo);
} catch (IOException ioe) {
// Ignore missing metainfo file for now
log.info("Missing metainfo {}", ioe.getMessage());
} catch (BadConfigException bce) {
log.info("Bad Configuration {}", bce.getMessage());
}
}
return metaInfo;
}
}

130
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
View File

@ -1,130 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;
class AgentLaunchParameter {
public static final Logger log =
LoggerFactory.getLogger(AgentLaunchParameter.class);
private static final String DEFAULT_PARAMETER = "";
private static final String ANY_COMPONENT = "ANY";
private static final String NONE_VALUE = "NONE";
private final Map<String, CommandTracker> launchParameterTracker;
public AgentLaunchParameter(String parameters) {
launchParameterTracker = parseExpectedLaunchParameters(parameters);
}
/**
* Get command for the component type
*
* @param componentGroup
*
* @return
*/
public String getNextLaunchParameter(String componentGroup) {
if (launchParameterTracker != null) {
if (launchParameterTracker.containsKey(componentGroup)
|| launchParameterTracker.containsKey(ANY_COMPONENT)) {
synchronized (this) {
CommandTracker indexTracker = null;
if (launchParameterTracker.containsKey(componentGroup)) {
indexTracker = launchParameterTracker.get(componentGroup);
} else {
indexTracker = launchParameterTracker.get(ANY_COMPONENT);
}
return indexTracker.getNextCommand();
}
}
}
return DEFAULT_PARAMETER;
}
/**
* Parse launch parameters of the form ANY:PARAM_FOR_FIRST:PARAM_FOR_SECOND:...:PARAM_FOR_REST|HBASE_MASTER:...
*
* E.g. ANY:DO_NOT_REGISTER:DO_NOT_HEARTBEAT:NONE For any container, first one gets DO_NOT_REGISTER second one gets
* DO_NOT_HEARTBEAT, then all of the rest get nothing
*
* E.g. HBASE_MASTER:FAIL_AFTER_START:NONE For HBASE_MASTER, first one gets FAIL_AFTER_START then "" for all
*
* @param launchParameters
*
* @return
*/
Map<String, CommandTracker> parseExpectedLaunchParameters(String launchParameters) {
Map<String, CommandTracker> trackers = null;
if (launchParameters != null && launchParameters.length() > 0) {
String[] componentSpecificParameters = launchParameters.split(Pattern.quote("|"));
for (String componentSpecificParameter : componentSpecificParameters) {
if (componentSpecificParameter.length() != 0) {
String[] parameters = componentSpecificParameter.split(Pattern.quote(":"));
if (parameters.length > 1 && parameters[0].length() > 0) {
for (int index = 1; index < parameters.length; index++) {
if (parameters[index].equals(NONE_VALUE)) {
parameters[index] = DEFAULT_PARAMETER;
}
}
if (trackers == null) {
trackers = new HashMap<String, CommandTracker>(10);
}
String componentName = parameters[0];
CommandTracker tracker = new CommandTracker(Arrays.copyOfRange(parameters, 1, parameters.length));
trackers.put(componentName, tracker);
}
}
}
}
return trackers;
}
class CommandTracker {
private final int maxIndex;
private final String[] launchCommands;
private int currentIndex;
CommandTracker(String[] launchCommands) {
this.currentIndex = 0;
this.maxIndex = launchCommands.length - 1;
this.launchCommands = launchCommands;
}
String getNextCommand() {
String retVal = launchCommands[currentIndex];
if (currentIndex != maxIndex) {
currentIndex++;
}
return retVal;
}
}
}

47
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderFactory.java
View File

@ -1,47 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
import org.apache.hadoop.conf.Configuration;
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.SliderProviderFactory;
public class AgentProviderFactory extends SliderProviderFactory {
public static final String CLASSNAME =
"org.apache.slider.providers.agent.AgentProviderFactory";
public AgentProviderFactory() {
}
public AgentProviderFactory(Configuration conf) {
super(conf);
}
@Override
public AbstractClientProvider createClientProvider() {
return new AgentClientProvider(getConf());
}
@Override
public ProviderService createServerProvider() {
return new AgentProviderService();
}
}

2850
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
View File

File diff suppressed because it is too large Load Diff

38
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
View File

@ -1,38 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
import org.apache.slider.providers.ProviderRole;
import java.util.ArrayList;
import java.util.List;
public class AgentRoles {
/**
* List of roles Agent provider does not have any roles by default. All roles are read from the application
* specification.
*/
protected static final List<ProviderRole> ROLES =
new ArrayList<ProviderRole>();
public static List<ProviderRole> getRoles() {
return ROLES;
}
}

150
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
View File

@ -1,150 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.agent.application.metadata.AbstractMetainfoParser;
import org.apache.slider.providers.agent.application.metadata.AddonPackageMetainfoParser;
import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
import org.apache.slider.providers.agent.application.metadata.DefaultConfigParser;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
/**
*
*/
public class AgentUtils {
private static final Logger log = LoggerFactory.getLogger(AgentUtils.class);
public static Metainfo getApplicationMetainfoFromSummaryFile(
SliderFileSystem fileSystem, String metainfoPath, boolean metainfoForAddon) {
FileSystem fs = fileSystem.getFileSystem();
Path appPathXML = new Path(metainfoPath + ".metainfo.xml");
Path appPathJson = new Path(metainfoPath + ".metainfo.json");
Path appPathUsed = null;
try {
FSDataInputStream appStream = null;
if (fs.exists(appPathXML)) {
appPathUsed = appPathXML;
appStream = fs.open(appPathXML);
return parseMetainfo(appStream, metainfoForAddon, "xml");
} else if (fs.exists(appPathJson)) {
appPathUsed = appPathJson;
appStream = fs.open(appPathJson);
return parseMetainfo(appStream, metainfoForAddon, "json");
}
} catch (IOException e) {
log.info("Failed to get metainfo from summary file {} - {}", appPathUsed,
e.getMessage());
log.debug("Failed to get metainfo", e);
}
return null;
}
public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
String metainfoPath, boolean metainfoForAddon) throws IOException,
BadConfigException {
log.info("Reading metainfo at {}", metainfoPath);
Metainfo metainfo = getApplicationMetainfoFromSummaryFile(fileSystem,
metainfoPath, metainfoForAddon);
if (metainfo != null) {
log.info("Got metainfo from summary file");
return metainfo;
}
FileSystem fs = fileSystem.getFileSystem();
Path appPath = new Path(metainfoPath);
InputStream metainfoJsonStream = SliderUtils.getApplicationResourceInputStream(
fs, appPath, "metainfo.json");
if (metainfoJsonStream == null) {
InputStream metainfoXMLStream = SliderUtils.getApplicationResourceInputStream(
fs, appPath, "metainfo.xml");
if (metainfoXMLStream != null) {
metainfo = parseMetainfo(metainfoXMLStream, metainfoForAddon, "xml");
}
} else {
metainfo = parseMetainfo(metainfoJsonStream, metainfoForAddon, "json");
}
if (metainfo == null) {
log.error("metainfo is unavailable at {}.", metainfoPath);
throw new FileNotFoundException("metainfo.xml/json is required in app package. " +
appPath);
}
return metainfo;
}
private static Metainfo parseMetainfo(InputStream stream,
boolean metainfoForAddon, String type) throws IOException {
AbstractMetainfoParser metainfoParser = null;
if (metainfoForAddon) {
metainfoParser = new AddonPackageMetainfoParser();
} else {
metainfoParser = new MetainfoParser();
}
if (type.equals("xml")) {
return metainfoParser.fromXmlStream(stream);
} else if (type.equals("json")) {
return metainfoParser.fromJsonStream(stream);
}
return null;
}
static DefaultConfig getDefaultConfig(SliderFileSystem fileSystem,
String appDef, String configFileName)
throws IOException {
// this is the path inside the zip file
String fileToRead = "configuration/" + configFileName;
log.info("Reading default config file {} at {}", fileToRead, appDef);
InputStream configStream = SliderUtils.getApplicationResourceInputStream(
fileSystem.getFileSystem(), new Path(appDef), fileToRead);
if (configStream == null) {
log.error("{} is unavailable at {}.", fileToRead, appDef);
throw new IOException("Expected config file " + fileToRead + " is not available.");
}
return new DefaultConfigParser().parse(configStream);
}
static String getMetainfoComponentName(String roleGroup,
ConfTreeOperations appConf) throws BadConfigException {
String prefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, null);
if (prefix == null) {
return roleGroup;
}
if (!roleGroup.startsWith(prefix)) {
throw new BadConfigException("Component " + roleGroup + " doesn't start" +
" with prefix " + prefix);
}
return roleGroup.substring(prefix.length());
}
}

59
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/Command.java
View File

@ -1,59 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
/** The states a component instance can be. */
public enum Command {
NOP, // do nothing
INSTALL, // Install the component
INSTALL_ADDON, // Install add on packages if any
START, // Start the component
STOP, // Stop the component
UPGRADE, // The component will undergo upgrade
TERMINATE; // Send terminate signal to agent
public static Command getCommand(String commandVal) {
if (commandVal.equals(Command.START.toString())) {
return Command.START;
}
if (commandVal.equals(Command.INSTALL.toString())) {
return Command.INSTALL;
}
if (commandVal.equals(Command.STOP.toString())) {
return Command.STOP;
}
if (commandVal.equals(Command.UPGRADE.toString())) {
return Command.UPGRADE;
}
if (commandVal.equals(Command.TERMINATE.toString())) {
return Command.TERMINATE;
}
return Command.NOP;
}
public static String transform(Command command, boolean isUpgrade) {
switch (command) {
case STOP:
return isUpgrade ? "UPGRADE_STOP" : command.name();
default:
return command.name();
}
}
}

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java
View File

@ -1,40 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
/** Command results. */
public enum CommandResult {
IN_PROGRESS, // Command is in progress
COMPLETED, // Command has successfully completed
FAILED; // Command has failed
public static CommandResult getCommandResult(String commandResVal) {
if (commandResVal.equals(CommandResult.COMPLETED.toString())) {
return CommandResult.COMPLETED;
}
if (commandResVal.equals(CommandResult.FAILED.toString())) {
return CommandResult.FAILED;
}
if (commandResVal.equals(CommandResult.IN_PROGRESS.toString())) {
return CommandResult.IN_PROGRESS;
}
throw new IllegalArgumentException("Unrecognized value " + commandResVal);
}
}

225
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
View File

@ -1,225 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.providers.agent.application.metadata.CommandOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
/**
* Stores the command dependency order for all components in a service. <commandOrder>
* <command>SUPERVISOR-START</command> <requires>NIMBUS-STARTED</requires> </commandOrder> Means, SUPERVISOR START
* requires NIMBUS to be STARTED
*/
public class ComponentCommandOrder {
public static final Logger log =
LoggerFactory.getLogger(ComponentCommandOrder.class);
private static char SPLIT_CHAR = '-';
Map<Command, Map<String, List<ComponentState>>> dependencies =
new HashMap<Command, Map<String, List<ComponentState>>>();
Map<String, Collection<String>> prefixRoleMap = new HashMap<>();
Map<String, String> rolePrefixMap = new HashMap<>();
public ComponentCommandOrder() {}
public ComponentCommandOrder(List<CommandOrder> commandOrders,
ConfTreeOperations resources) {
mergeCommandOrders(commandOrders, resources);
}
void mergeCommandOrders(List<CommandOrder> commandOrders,
ConfTreeOperations resources) {
for (String component : resources.getComponentNames()) {
String prefix = SliderUtils.trimPrefix(
resources.getComponentOpt(component, ROLE_PREFIX, null));
if (prefix != null) {
rolePrefixMap.put(component, prefix);
if (!prefixRoleMap.containsKey(prefix)) {
prefixRoleMap.put(prefix, new HashSet<String>());
}
prefixRoleMap.get(prefix).add(component);
}
}
if (commandOrders != null && commandOrders.size() > 0) {
for (CommandOrder commandOrder : commandOrders) {
ComponentCommand componentCmd = getComponentCommand(
commandOrder.getCommand(), resources);
String requires = commandOrder.getRequires();
List<ComponentState> requiredStates = parseRequiredStates(requires,
resources);
if (requiredStates.size() > 0) {
Map<String, List<ComponentState>> compDep = dependencies.get(componentCmd.command);
if (compDep == null) {
compDep = new HashMap<>();
dependencies.put(componentCmd.command, compDep);
}
List<ComponentState> requirements = compDep.get(componentCmd.componentName);
if (requirements == null) {
requirements = new ArrayList<>();
compDep.put(componentCmd.componentName, requirements);
}
requirements.addAll(requiredStates);
}
}
}
}
private List<ComponentState> parseRequiredStates(String requires,
ConfTreeOperations resources) {
if (requires == null || requires.length() < 2) {
throw new IllegalArgumentException("Input cannot be null and must contain component and state.");
}
String[] componentStates = requires.split(",");
List<ComponentState> retList = new ArrayList<ComponentState>();
for (String componentStateStr : componentStates) {
retList.add(getComponentState(componentStateStr, resources));
}
return retList;
}
private ComponentCommand getComponentCommand(String compCmdStr,
ConfTreeOperations resources) {
if (compCmdStr == null || compCmdStr.trim().length() < 2) {
throw new IllegalArgumentException("Input cannot be null and must contain component and command.");
}
compCmdStr = compCmdStr.trim();
int splitIndex = compCmdStr.lastIndexOf(SPLIT_CHAR);
if (splitIndex == -1 || splitIndex == 0 || splitIndex == compCmdStr.length() - 1) {
throw new IllegalArgumentException("Input does not appear to be well-formed.");
}
String compStr = compCmdStr.substring(0, splitIndex);
String cmdStr = compCmdStr.substring(splitIndex + 1);
if (resources.getComponent(compStr) == null && !prefixRoleMap.containsKey(compStr)) {
throw new IllegalArgumentException("Component " + compStr + " specified" +
" in command order does not exist");
}
Command cmd = Command.valueOf(cmdStr);
if (cmd != Command.START) {
throw new IllegalArgumentException("Dependency order can only be specified for START.");
}
return new ComponentCommand(compStr, cmd);
}
private ComponentState getComponentState(String compStStr,
ConfTreeOperations resources) {
if (compStStr == null || compStStr.trim().length() < 2) {
throw new IllegalArgumentException("Input cannot be null.");
}
compStStr = compStStr.trim();
int splitIndex = compStStr.lastIndexOf(SPLIT_CHAR);
if (splitIndex == -1 || splitIndex == 0 || splitIndex == compStStr.length() - 1) {
throw new IllegalArgumentException("Input does not appear to be well-formed.");
}
String compStr = compStStr.substring(0, splitIndex);
String stateStr = compStStr.substring(splitIndex + 1);
if (resources.getComponent(compStr) == null && !prefixRoleMap.containsKey(compStr)) {
throw new IllegalArgumentException("Component " + compStr + " specified" +
" in command order does not exist");
}
State state = State.valueOf(stateStr);
if (state != State.STARTED && state != State.INSTALLED) {
throw new IllegalArgumentException("Dependency order can only be specified against STARTED/INSTALLED.");
}
return new ComponentState(compStr, state);
}
// dependency is still on component level, but not package level
// so use component name to check dependency, not component-package
public boolean canExecute(String component, Command command, Collection<ComponentInstanceState> currentStates) {
if (!dependencies.containsKey(command)) {
return true;
}
List<ComponentState> required = new ArrayList<>();
if (dependencies.get(command).containsKey(component)) {
required.addAll(dependencies.get(command).get(component));
}
String prefix = rolePrefixMap.get(component);
if (prefix != null && dependencies.get(command).containsKey(prefix)) {
required.addAll(dependencies.get(command).get(prefix));
}
for (ComponentState stateToMatch : required) {
for (ComponentInstanceState currState : currentStates) {
log.debug("Checking schedule {} {} against dependency {} is {}",
component, command, currState.getComponentName(), currState.getState());
if (currState.getComponentName().equals(stateToMatch.componentName) ||
(prefixRoleMap.containsKey(stateToMatch.componentName) &&
prefixRoleMap.get(stateToMatch.componentName).contains(currState.getComponentName()))) {
if (currState.getState() != stateToMatch.state) {
if (stateToMatch.state == State.STARTED) {
log.info("Cannot schedule {} {} as dependency {} is {}",
component, command, currState.getComponentName(), currState.getState());
return false;
} else {
//state is INSTALLED
if (currState.getState() != State.STARTING && currState.getState() != State.STARTED) {
log.info("Cannot schedule {} {} as dependency {} is {}",
component, command, currState.getComponentName(), currState.getState());
return false;
}
}
}
}
}
}
return true;
}
static class ComponentState {
public String componentName;
public State state;
public ComponentState(String componentName, State state) {
this.componentName = componentName;
this.state = state;
}
}
static class ComponentCommand {
public String componentName;
public Command command;
public ComponentCommand(String componentName, Command command) {
this.componentName = componentName;
this.command = command;
}
}
}

340
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
View File

@ -1,340 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
import java.util.Map;
import java.util.TreeMap;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.slider.providers.agent.application.metadata.Component;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** This class implements a simple state machine for component instances. */
public class ComponentInstanceState {
public static final Logger log =
LoggerFactory.getLogger(ComponentInstanceState.class);
private static int MAX_FAILURE_TOLERATED = 3;
private static String INVALID_TRANSITION_ERROR =
"Result %s for command %s is not expected for component %s in state %s.";
private final String componentName;
private final ContainerId containerId;
private final String containerIdAsString;
private final String applicationId;
private State state = State.INIT;
private State targetState = State.STARTED;
private int failuresSeen = 0;
private Boolean configReported = false;
private long lastHeartbeat = 0;
private String ip;
private String hostname;
private ContainerState containerState;
private Map<String, State> pkgStatuses;
private String nextPkgToInstall;
private boolean stopInitiated;
public ComponentInstanceState(String componentName,
ContainerId containerId,
String applicationId) {
this(componentName, containerId, applicationId,
new TreeMap<String, State>());
}
public ComponentInstanceState(String componentName,
ContainerId containerId,
String applicationId, Map<String, State> pkgStatuses) {
this.componentName = componentName;
this.containerId = containerId;
this.containerIdAsString = containerId.toString();
this.applicationId = applicationId;
this.containerState = ContainerState.INIT;
this.lastHeartbeat = System.currentTimeMillis();
this.pkgStatuses = pkgStatuses;
}
public String getComponentName() {
return componentName;
}
public Boolean getConfigReported() {
return configReported;
}
public void setConfigReported(Boolean configReported) {
this.configReported = configReported;
}
public ContainerState getContainerState() {
return containerState;
}
public void setContainerState(ContainerState containerState) {
this.containerState = containerState;
}
public long getLastHeartbeat() {
return lastHeartbeat;
}
/**
* Update the heartbeat, and change container state
* to mark as healthy if appropriate
* @param heartbeatTime last time the heartbeat was seen
* @return the current container state
*/
public ContainerState heartbeat(long heartbeatTime) {
this.lastHeartbeat = heartbeatTime;
if(containerState == ContainerState.UNHEALTHY ||
containerState == ContainerState.INIT) {
containerState = ContainerState.HEALTHY;
}
return containerState;
}
public ContainerId getContainerId() {
return containerId;
}
public void commandIssued(Command command) {
commandIssued(command, false);
}
public void commandIssued(Command command, boolean isInUpgradeMode) {
Command expected = getNextCommand(isInUpgradeMode);
if (expected != command) {
throw new IllegalArgumentException("Command " + command + " is not allowed in state " + state);
}
if (expected == Command.INSTALL_ADDON) {
// for add on packages, the pkg must be nextPkgToInstall
State currentState = pkgStatuses.get(nextPkgToInstall);
log.debug("Command issued: component: {} is in {}", componentName,
currentState);
State nextState = currentState.getNextState(command);
pkgStatuses.put(nextPkgToInstall, nextState);
log.debug("Command issued: component: {} is now in {}", componentName,
nextState);
} else {
// for master package
state = state.getNextState(command);
}
}
public void applyCommandResult(CommandResult result, Command command,
String pkg) {
// if the heartbeat is for a package
// update that package's state in the component status
// and don't bother with the master pkg
if (StringUtils.isNotEmpty(pkg)
&& !Component.MASTER_PACKAGE_NAME.equals(pkg)) {
log.debug("This result is for component: {} pkg: {}", componentName, pkg);
State previousPkgState = pkgStatuses.get(pkg);
log.debug("Currently component: {} pkg: {} is in state: {}",
componentName, pkg, previousPkgState.toString());
State nextPkgState = previousPkgState.getNextState(result);
pkgStatuses.put(pkg, nextPkgState);
log.debug("Component: {} pkg: {} next state: {}", componentName, pkg,
nextPkgState);
} else {
log.debug("This result is for component: {} master package",
componentName);
applyCommandResult(result, command);
}
}
public void applyCommandResult(CommandResult result, Command command) {
if (!this.state.couldHaveIssued(command)) {
throw new IllegalStateException("Invalid command " + command + " for state " + this.state);
}
try {
if (result == CommandResult.FAILED) {
failuresSeen++;
} else if (result == CommandResult.COMPLETED) {
failuresSeen = 0;
}
state = state.getNextState(result);
} catch (IllegalArgumentException e) {
String message = String.format(INVALID_TRANSITION_ERROR,
result.toString(),
command.toString(),
componentName,
state.toString());
log.warn(message);
throw new IllegalStateException(message);
}
}
public boolean hasPendingCommand() {
if (state.canIssueCommands() &&
state != targetState &&
failuresSeen < MAX_FAILURE_TOLERATED) {
return true;
}
return false;
}
public Command getNextCommand() {
return getNextCommand(false);
}
public Command getNextCommand(boolean isInUpgradeMode) {
if (!hasPendingCommand()) {
nextPkgToInstall = null;
return Command.NOP;
}
log.debug("In getNextCommand, checking for component: {} ", componentName);
// if the master pkg is just installed, check if any add on pkg need to be
// installed
nextPkgToInstall = null;
if (state == State.INSTALLED) {
for (Map.Entry<String, State> pkgStatus : pkgStatuses.entrySet()) {
String pkg = pkgStatus.getKey();
State pkgState = pkgStatus.getValue();
log.debug("In getNextCommand, pkg: {} is in {}", pkg, pkgState);
if (pkgState == State.INSTALLING) {
// first check if any pkg is install in progress, if so, wait
// so we don't need to do anything, just return NOP
log.debug("In getNextCommand, pkg: {} we are issuing NOP", pkg);
nextPkgToInstall = pkg;
return Command.NOP;
} else if (pkgState == State.INIT) {
// temporarily storing pkg here
// in case no pkg in 'installing' state
// will return the package to install
nextPkgToInstall = pkg;
}
}
// when we reach here, no pkg is in 'installing' state
if (nextPkgToInstall != null) {
// nextPkgToInstall != null means some pkg is in INIT state
// issue 'install' to the pkg we have stored in nextPkgToInstall
log.debug("In getNextCommand, pkg: {} we are issuing install addon",
nextPkgToInstall);
return Command.INSTALL_ADDON;
}
}
return this.state.getSupportedCommand(isInUpgradeMode, stopInitiated);
}
public State getState() {
return state;
}
@VisibleForTesting
protected void setState(State state) {
this.state = state;
}
public State getTargetState() {
return targetState;
}
public void setTargetState(State targetState) {
this.targetState = targetState;
}
public String getNextPkgToInstall() {
return nextPkgToInstall;
}
public boolean isStopInitiated() {
return stopInitiated;
}
public void setStopInitiated(boolean stopInitiated) {
this.stopInitiated = stopInitiated;
}
@Override
public int hashCode() {
int hashCode = 1;
hashCode = hashCode ^ (componentName != null ? componentName.hashCode() : 0);
hashCode = hashCode ^ (containerIdAsString != null ? containerIdAsString.hashCode() : 0);
hashCode = hashCode ^ (applicationId != null ? applicationId.hashCode() : 0);
return hashCode;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ComponentInstanceState that = (ComponentInstanceState) o;
if (this.componentName != null ?
!this.componentName.equals(that.componentName) : this.componentName != null) {
return false;
}
if (this.containerIdAsString != null ?
!this.containerIdAsString.equals(that.containerIdAsString) : this.containerIdAsString != null) {
return false;
}
if (this.applicationId != null ?
!this.applicationId.equals(that.applicationId) : this.applicationId != null) {
return false;
}
return true;
}
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder("ComponentInstanceState{");
sb.append("containerIdAsString='").append(containerIdAsString).append('\'');
sb.append(", state=").append(state);
sb.append(", failuresSeen=").append(failuresSeen);
sb.append(", lastHeartbeat=").append(lastHeartbeat);
sb.append(", containerState=").append(containerState);
sb.append(", componentName='").append(componentName).append('\'');
sb.append(", ip=").append(ip);
sb.append(", hostname='").append(hostname).append('\'');
sb.append('}');
return sb.toString();
}
public String getIp() {
return ip;
}
public void setIp(String ip) {
this.ip = ip;
}
public String getHostname() {
return hostname;
}
public void setHostname(String hostname) {
this.hostname = hostname;
}
}

127
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentTagProvider.java
View File

@ -1,127 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
import org.apache.slider.common.tools.SliderUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.ConcurrentHashMap;
/** A simple tag provider that attempts to associate tags from 1-N to all container of a component */
public class ComponentTagProvider {
private static final Logger log = LoggerFactory.getLogger(ComponentTagProvider.class);
private static String FREE = "free";
private final ConcurrentHashMap<String, ConcurrentHashMap<String, String>> allTags;
public ComponentTagProvider() {
allTags = new ConcurrentHashMap<String, ConcurrentHashMap<String, String>>();
}
/**
* Record an assigned tag to a container
*
* @param component
* @param containerId
* @param tag
*/
public void recordAssignedTag(String component, String containerId, String tag) {
if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) {
Integer key = null;
try {
key = Integer.valueOf(tag);
} catch (NumberFormatException nfe) {
//ignore
}
if (key != null && key > 0) {
ConcurrentHashMap<String, String> compTags = getComponentSpecificTags(component);
synchronized (compTags) {
for (int index = 1; index <= key.intValue(); index++) {
String tempKey = new Integer(index).toString();
if (!compTags.containsKey(tempKey)) {
compTags.put(tempKey, FREE);
}
}
compTags.put(key.toString(), containerId);
}
}
}
}
/**
* Get a tag for container
*
* @param component
* @param containerId
*
* @return
*/
public String getTag(String component, String containerId) {
if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) {
ConcurrentHashMap<String, String> compTags = getComponentSpecificTags(component);
synchronized (compTags) {
for (String key : compTags.keySet()) {
if (compTags.get(key).equals(containerId)) {
return key;
}
}
for (String key : compTags.keySet()) {
if (compTags.get(key).equals(FREE)) {
compTags.put(key, containerId);
return key;
}
}
String newKey = new Integer(compTags.size() + 1).toString();
compTags.put(newKey, containerId);
return newKey;
}
}
return "";
}
/**
* Release a tag associated with a container
*
* @param component
* @param containerId
*/
public void releaseTag(String component, String containerId) {
if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) {
ConcurrentHashMap<String, String> compTags = allTags.get(component);
if (compTags != null) {
synchronized (compTags) {
for (String key : compTags.keySet()) {
if (compTags.get(key).equals(containerId)) {
compTags.put(key, FREE);
}
}
}
}
}
}
private ConcurrentHashMap<String, String> getComponentSpecificTags(String component) {
if (!allTags.containsKey(component)) {
synchronized (allTags) {
if (!allTags.containsKey(component)) {
allTags.put(component, new ConcurrentHashMap<String, String>());
}
}
}
return allTags.get(component);
}
}

41
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java
View File

@ -1,41 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
/** The states a component instance can be. */
public enum ContainerState {
INIT, // Container is not net activated
HEALTHY, // Agent is heartbeating
UNHEALTHY, // Container is unhealthy - no heartbeat for some interval
HEARTBEAT_LOST; // Container is lost - request a new instance
/**
* Indicates whether or not it is a valid state to produce a command.
*
* @return true if command can be issued for this state.
*/
public boolean canIssueCommands() {
switch (this) {
case HEALTHY:
return true;
default:
return false;
}
}
}

130
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
View File

@ -1,130 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
/** Monitors the container state and heartbeats. */
public class HeartbeatMonitor implements Runnable {
protected static final Logger log =
LoggerFactory.getLogger(HeartbeatMonitor.class);
private final int threadWakeupInterval; //1 minute
private final AgentProviderService provider;
private volatile boolean shouldRun = true;
private Thread monitorThread = null;
public HeartbeatMonitor(AgentProviderService provider, int threadWakeupInterval) {
this.provider = provider;
this.threadWakeupInterval = threadWakeupInterval;
}
public void shutdown() {
shouldRun = false;
}
public void start() {
log.info("Starting heartbeat monitor with interval {}", threadWakeupInterval);
monitorThread = new Thread(this);
monitorThread.start();
}
void join(long millis) throws InterruptedException {
if (isAlive()) {
monitorThread.join(millis);
}
}
public boolean isAlive() {
return monitorThread != null && monitorThread.isAlive();
}
@Override
public void run() {
while (shouldRun) {
try {
log.debug("Putting monitor to sleep for " + threadWakeupInterval + " " +
"milliseconds");
Thread.sleep(threadWakeupInterval);
doWork(System.currentTimeMillis());
} catch (InterruptedException ex) {
log.warn("Scheduler thread is interrupted going to stop", ex);
shouldRun = false;
} catch (Exception ex) {
log.warn("Exception received", ex);
} catch (Throwable t) {
log.warn("ERROR", t);
}
}
}
/**
* Every interval the current state of the container are checked. If the state is INIT or HEALTHY and no HB are
* received in last check interval they are marked as UNHEALTHY. INIT is when the agent is started but it did not
* communicate at all. HEALTHY being the AM has received heartbeats. After an interval as UNHEALTHY the container is
* declared unavailable
* @param now current time in milliseconds ... tests can set this explicitly
*/
@VisibleForTesting
public void doWork(long now) {
Map<String, ComponentInstanceState> componentStatuses = provider.getComponentStatuses();
if (componentStatuses != null) {
for (String containerLabel : componentStatuses.keySet()) {
ComponentInstanceState componentInstanceState = componentStatuses.get(containerLabel);
long timeSinceLastHeartbeat = now - componentInstanceState.getLastHeartbeat();
if (timeSinceLastHeartbeat > threadWakeupInterval) {
switch (componentInstanceState.getContainerState()) {
case INIT:
case HEALTHY:
componentInstanceState.setContainerState(ContainerState.UNHEALTHY);
log.warn(
"Component {} marked UNHEALTHY. Last heartbeat received at {} approx. {} ms. back.",
componentInstanceState,
componentInstanceState.getLastHeartbeat(),
timeSinceLastHeartbeat);
break;
case UNHEALTHY:
if (timeSinceLastHeartbeat > threadWakeupInterval * 2) {
componentInstanceState.setContainerState(
ContainerState.HEARTBEAT_LOST);
log.warn(
"Component {} marked HEARTBEAT_LOST. Last heartbeat received at {} approx. {} ms. back.",
componentInstanceState, componentInstanceState.getLastHeartbeat(),
timeSinceLastHeartbeat);
ContainerId containerId =
componentInstanceState.getContainerId();
provider.lostContainer(containerLabel, containerId);
}
break;
case HEARTBEAT_LOST:
// unexpected case
log.warn("Heartbeat from lost component: {}", componentInstanceState);
break;
}
}
}
}
}
}

199
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/State.java
View File

@ -1,199 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
/** The states a component instance can be. */
public enum State {
INIT, // Not installed
INSTALLING, // Being installed
INSTALLED, // Installed (or stopped)
STARTING, // Starting
STARTED, // Started
INSTALL_FAILED, // Install failed, start failure in INSTALLED
UPGRADING, // Undergoing upgrade, perform necessary pre-upgrade steps
UPGRADED, // Pre-upgrade steps completed
STOPPING, // Stop has been issued
STOPPED, // Agent has stopped
TERMINATING; // Terminate signal to ask the agent to kill itself
// No need for state TERMINATED (as the agent is dead by then)
/**
* Indicates whether or not it is a valid state to produce a command.
*
* @return true if command can be issued for this state.
*/
public boolean canIssueCommands() {
switch (this) {
case INSTALLING:
case STARTING:
case UPGRADING:
case STOPPING:
case TERMINATING:
return false;
default:
return true;
}
}
/**
* Returns valid command in this state.
*
* @return command allowed in this state.
*/
public Command getSupportedCommand() {
return getSupportedCommand(false);
}
public Command getSupportedCommand(boolean isInUpgradeMode) {
return getSupportedCommand(isInUpgradeMode, false);
}
public Command getSupportedCommand(boolean isInUpgradeMode,
boolean stopInitiated) {
switch (this) {
case INIT:
case INSTALL_FAILED:
return Command.INSTALL;
case INSTALLED:
return Command.START;
case STARTED:
return isInUpgradeMode ? Command.UPGRADE : (stopInitiated) ? Command.STOP
: Command.NOP;
case UPGRADED:
return Command.STOP;
case STOPPED:
return Command.TERMINATE;
default:
return Command.NOP;
}
}
/**
* Returns next state based on the command result.
*
* @return next state.
*/
public State getNextState(CommandResult result) throws IllegalArgumentException {
switch (result) {
case IN_PROGRESS:
if (this == State.INSTALLING || this == State.STARTING
|| this == State.UPGRADING || this == State.STOPPING
|| this == State.TERMINATING) {
return this;
} else {
throw new IllegalArgumentException(result + " is not valid for " + this);
}
case COMPLETED:
if (this == State.INSTALLING) {
return State.INSTALLED;
} else if (this == State.STARTING) {
return State.STARTED;
} else if (this == State.UPGRADING) {
return State.UPGRADED;
} else if (this == State.STOPPING) {
return State.STOPPED;
} else {
throw new IllegalArgumentException(result + " is not valid for " + this);
}
case FAILED:
if (this == State.INSTALLING) {
return State.INSTALL_FAILED;
} else if (this == State.STARTING) {
return State.INSTALLED;
} else if (this == State.UPGRADING) {
// if pre-upgrade failed, force stop now, so mark it upgraded
// what other options can be exposed to app owner?
return State.UPGRADED;
} else if (this == State.STOPPING) {
// if stop fails, force mark it stopped (and let container terminate)
return State.STOPPED;
} else if (this == State.STOPPED) {
// if in stopped state, force mark it as terminating
return State.TERMINATING;
} else {
throw new IllegalArgumentException(result + " is not valid for " + this);
}
default:
throw new IllegalArgumentException("Bad command result " + result);
}
}
/**
* Returns next state based on the command.
*
* @return next state.
*/
public State getNextState(Command command) throws IllegalArgumentException {
switch (command) {
case INSTALL:
if (this == State.INIT || this == State.INSTALL_FAILED) {
return State.INSTALLING;
} else {
throw new IllegalArgumentException(command + " is not valid for " + this);
}
case INSTALL_ADDON:
if (this == State.INIT || this == State.INSTALL_FAILED) {
return State.INSTALLING;
} else {
throw new IllegalArgumentException(command + " is not valid for " + this);
}
case START:
if (this == State.INSTALLED) {
return State.STARTING;
} else {
throw new IllegalArgumentException(command + " is not valid for " + this);
}
case UPGRADE:
if (this == State.STARTED) {
return State.UPGRADING;
} else {
throw new IllegalArgumentException(command + " is not valid for " + this);
}
case STOP:
if (this == State.STARTED || this == State.UPGRADED) {
return State.STOPPING;
} else {
throw new IllegalArgumentException(command + " is not valid for " + this);
}
case TERMINATE:
if (this == State.STOPPED) {
return State.TERMINATING;
} else {
throw new IllegalArgumentException(command + " is not valid for " + this);
}
case NOP:
return this;
default:
throw new IllegalArgumentException("Bad command " + command);
}
}
public boolean couldHaveIssued(Command command) {
if ((this == State.INSTALLING && command == Command.INSTALL)
|| (this == State.STARTING && command == Command.START)
|| (this == State.UPGRADING && command == Command.UPGRADE)
|| (this == State.STOPPING
&& (command == Command.STOP || command == Command.NOP))
|| (this == State.TERMINATING && command == Command.TERMINATE)
) {
return true;
}
return false;
}
}

80
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java
View File

@ -1,80 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import java.util.ArrayList;
import java.util.List;
import org.codehaus.jackson.annotate.JsonProperty;
/**
* Component defined in master package metainfo.json
*/
public abstract class AbstractComponent implements Validate {
public static final String TYPE_STANDARD = "STANDARD";
public static final String TYPE_DOCKER = "DOCKER";
public static final String TYPE_PYTHON = "PYTHON";
public static final String CATEGORY_MASTER = "MASTER";
public static final String CATEGORY_SLAVE = "SLAVE";
public static final String CATEGORY_CLIENT = "CLIENT";
public static final String MASTER_PACKAGE_NAME = "MASTER";
protected String name;
protected CommandScript commandScript;
protected List<ComponentCommand> commands = new ArrayList<>();
public AbstractComponent() {
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public CommandScript getCommandScript() {
return commandScript;
}
public void addCommandScript(CommandScript commandScript) {
this.commandScript = commandScript;
}
@JsonProperty("commands")
public List<ComponentCommand> getCommands() {
return commands;
}
public void setCommands(List<ComponentCommand> commands) {
this.commands = commands;
}
public void addCommand(ComponentCommand command) {
commands.add(command);
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("{");
sb.append("\n\"name\": ").append(name);
sb.append(",\n\"commandScript\" :").append(commandScript);
sb.append('}');
return sb.toString();
}
}

130
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java
View File

@ -1,130 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.commons.digester.Digester;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
/**
* This abstract class provide common functionality to parse metainfo.json for
* either master package or add on packages.
*/
public abstract class AbstractMetainfoParser {
protected final GsonBuilder gsonBuilder = new GsonBuilder();
protected final Gson gson;
private static final Logger log = LoggerFactory
.getLogger(AbstractMetainfoParser.class);
public AbstractMetainfoParser() {
gson = gsonBuilder.create();
}
/**
* Convert to a JSON string
*
* @return a JSON string description
*
* @throws IOException Problems mapping/writing the object
*/
public String toJsonString(Metainfo metaInfo) throws IOException {
return gson.toJson(metaInfo);
}
/**
* Convert from JSON
*
* @param json input
*
* @return the parsed JSON
*
* @throws IOException IO
*/
public Metainfo fromJsonString(String json)
throws IOException {
return gson.fromJson(json, Metainfo.class);
}
/**
* Parse metainfo from an IOStream
*
* @param is
*
* @return
*
* @throws IOException
*/
public Metainfo fromJsonStream(InputStream is) throws IOException {
log.debug("loading from xml stream");
StringWriter writer = new StringWriter();
IOUtils.copy(is, writer);
return fromJsonString(writer.toString());
}
/**
* Parse metainfo from an XML formatted IOStream
*
* @param metainfoStream
*
* @return
*
* @throws IOException
*/
public Metainfo fromXmlStream(InputStream metainfoStream) throws IOException {
log.debug("loading from xml stream");
Digester digester = new Digester();
digester.setValidating(false);
composeSchema(digester);
try {
return (Metainfo) digester.parse(metainfoStream);
} catch (IOException e) {
log.debug("IOException in metainfoparser during fromXmlStream: "
+ e.getMessage());
} catch (SAXException e) {
log.debug("SAXException in metainfoparser during fromXmlStream: "
+ e.getMessage());
} finally {
if (metainfoStream != null) {
metainfoStream.close();
}
}
return null;
}
/**
* Compose the schema for the metainfo
*
* @param Digester - The Digester object we passed in to compose the schema
*
* @return
*
* @throws IOException
*/
abstract protected void composeSchema(Digester digester);
}

69
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java
View File

@ -1,69 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.codehaus.jackson.annotate.JsonProperty;
import java.util.ArrayList;
import java.util.List;
/**
* Application type defined in the metainfo
*/
public abstract class AbstractMetainfoSchema implements Validate {
protected String name;
protected String comment;
protected String version;
protected List<ConfigFile> configFiles = new ArrayList<>();
public AbstractMetainfoSchema() {
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getComment() {
return comment;
}
public void setComment(String comment) {
this.comment = comment;
}
public String getVersion() {
return version;
}
public void setVersion(String version) {
this.version = version;
}
public void addConfigFile(ConfigFile configFile) {
this.configFiles.add(configFile);
}
@JsonProperty("configFiles")
public List<ConfigFile> getConfigFiles() {
return configFiles;
}
}

53
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java
View File

@ -1,53 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.commons.digester.Digester;
/**
*
*/
public class AddonPackageMetainfoParser extends AbstractMetainfoParser {
protected void composeSchema(Digester digester) {
digester.addObjectCreate("metainfo", Metainfo.class);
digester.addBeanPropertySetter("metainfo/schemaVersion");
digester.addObjectCreate("*/applicationPackage", ApplicationPackage.class);
digester.addBeanPropertySetter("*/applicationPackage/name");
digester.addBeanPropertySetter("*/applicationPackage/comment");
digester.addBeanPropertySetter("*/applicationPackage/version");
digester.addObjectCreate("*/component", ComponentsInAddonPackage.class);
digester.addBeanPropertySetter("*/component/name");
digester.addSetNext("*/component", "addComponent");
digester.addObjectCreate("*/commandScript", CommandScript.class);
digester.addBeanPropertySetter("*/commandScript/script");
digester.addBeanPropertySetter("*/commandScript/scriptType");
digester.addBeanPropertySetter("*/commandScript/timeout");
digester.addSetNext("*/commandScript", "addCommandScript");
digester.addObjectCreate("*/configFile", ConfigFile.class);
digester.addBeanPropertySetter("*/configFile/type");
digester.addBeanPropertySetter("*/configFile/fileName");
digester.addBeanPropertySetter("*/configFile/dictionaryName");
digester.addSetNext("*/configFile", "addConfigFile");
digester.addSetRoot("*/applicationPackage", "setApplicationPackage");
}
}

193
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
View File

@ -1,193 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.SliderException;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonProperty;
import java.util.ArrayList;
import java.util.List;
/**
* Application type defined in the metainfo
*/
public class Application extends AbstractMetainfoSchema {
String exportedConfigs;
List<ExportGroup> exportGroups = new ArrayList<>();
List<OSSpecific> osSpecifics = new ArrayList<>();
List<CommandOrder> commandOrders = new ArrayList<>();
List<Package> packages = new ArrayList<>();
private List<Component> components = new ArrayList<>();
public Application() {
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getComment() {
return comment;
}
public void setComment(String comment) {
this.comment = comment;
}
public String getVersion() {
return version;
}
public void setVersion(String version) {
this.version = version;
}
public String getExportedConfigs() {
return exportedConfigs;
}
public void setExportedConfigs(String exportedConfigs) {
this.exportedConfigs = exportedConfigs;
}
public void addConfigFile(ConfigFile configFile) {
this.configFiles.add(configFile);
}
@JsonProperty("configFiles")
public List<ConfigFile> getConfigFiles() {
return configFiles;
}
public void addComponent(Component component) {
components.add(component);
}
@JsonProperty("components")
public List<Component> getComponents() {
return components;
}
public void addExportGroup(ExportGroup exportGroup) {
exportGroups.add(exportGroup);
}
@JsonProperty("exportGroups")
public List<ExportGroup> getExportGroups() {
return exportGroups;
}
public void addOSSpecific(OSSpecific osSpecific) {
osSpecifics.add(osSpecific);
}
@JsonIgnore
public List<OSSpecific> getOSSpecifics() {
return osSpecifics;
}
public void addCommandOrder(CommandOrder commandOrder) {
commandOrders.add(commandOrder);
}
@JsonProperty("commandOrders")
public List<CommandOrder> getCommandOrders() {
return commandOrders;
}
public void addPackage(Package pkg) {
packages.add(pkg);
}
@JsonProperty("packages")
public List<Package> getPackages() {
return packages;
}
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder("{");
sb.append(",\n\"name\": ").append(name);
sb.append(",\n\"comment\": ").append(comment);
sb.append(",\n\"version\" :").append(version);
sb.append(",\n\"components\" : {");
for (Component component : components) {
sb.append("\n").append(component.toString());
}
sb.append("\n},");
sb.append('}');
return sb.toString();
}
public void validate(String version) throws SliderException {
if(SliderUtils.isUnset(version)) {
throw new BadCommandArgumentsException("schema version cannot be null");
}
Metainfo.checkNonNull(getName(), "name", "application");
Metainfo.checkNonNull(getVersion(), "version", "application");
if(getComponents().size() == 0) {
throw new SliderException("application must contain at least one component");
}
if(version.equals(Metainfo.VERSION_TWO_ZERO)) {
if(getPackages().size() > 0) {
throw new SliderException("packages is not supported in version " + version);
}
}
if(version.equals(Metainfo.VERSION_TWO_ONE)) {
if(getOSSpecifics().size() > 0) {
throw new SliderException("osSpecifics is not supported in version " + version);
}
}
for(CommandOrder co : getCommandOrders()) {
co.validate(version);
}
for(Component comp : getComponents()) {
comp.validate(version);
}
for(ConfigFile cf : getConfigFiles()) {
cf.validate(version);
}
for(ExportGroup eg : getExportGroups()) {
eg.validate(version);
}
for(Package pkg : getPackages()) {
pkg.validate(version);
}
for(OSSpecific os : getOSSpecifics()) {
os.validate(version);
}
}
}

69
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java
View File

@ -1,69 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import java.util.ArrayList;
import java.util.List;
import org.apache.slider.core.exceptions.SliderException;
public class ApplicationPackage extends AbstractMetainfoSchema{
private List<ComponentsInAddonPackage> components = new ArrayList<ComponentsInAddonPackage>();
public void addComponent(ComponentsInAddonPackage component) {
components.add(component);
}
// we must override getcomponent() as well. otherwise it is pointing to the
// overriden components of type List<Component>
public List<ComponentsInAddonPackage> getComponents(){
return this.components;
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("{");
sb.append("\n\"name\": ").append(name);
sb.append(",\n\"comment\": ").append(comment);
sb.append(",\n\"version\" :").append(version);
sb.append(",\n\"components\" : {");
for (ComponentsInAddonPackage component : components) {
sb.append("\n").append(component);
}
sb.append("\n},");
sb.append('}');
return sb.toString();
}
@Override
public void validate(String version) throws SliderException {
if (name == null || name.isEmpty()) {
throw new SliderException(
"Missing name in metainfo.json for add on packages");
}
if (components.isEmpty()) {
throw new SliderException(
"Missing components in metainfo.json for add on packages");
}
for (ComponentsInAddonPackage component : components) {
if (component.name == null || component.name.isEmpty()) {
throw new SliderException(
"Missing name of components in metainfo.json for add on packages");
}
}
}
}

61
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java
View File

@ -1,61 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.core.exceptions.SliderException;
/**
*
*/
public class CommandOrder implements Validate {
String command;
String requires;
public CommandOrder() {
}
public String getCommand() {
return command;
}
public void setCommand(String command) {
this.command = command;
}
public String getRequires() {
return requires;
}
public void setRequires(String requires) {
this.requires = requires;
}
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder("{");
sb.append(",\n\"command\": ").append(command);
sb.append(",\n\"requires\": ").append(requires);
sb.append('}');
return sb.toString();
}
public void validate(String version) throws SliderException {
Metainfo.checkNonNull(getCommand(), "command", "package");
Metainfo.checkNonNull(getRequires(), "requires", "package");
}
}

72
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java
View File

@ -1,72 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.core.exceptions.SliderException;
/**
* CommandScript that implements all component commands
*/
public class CommandScript implements Validate {
String script;
String scriptType;
long timeout;
public CommandScript() {
}
public String getScript() {
return script;
}
public void setScript(String script) {
this.script = script;
}
public String getScriptType() {
return scriptType;
}
public void setScriptType(String scriptType) {
this.scriptType = scriptType;
}
public long getTimeout() {
return timeout;
}
public void setTimeout(long timeout) {
this.timeout = timeout;
}
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder("{");
sb.append(",\n\"script\": ").append(script);
sb.append(",\n\"scriptType\": ").append(scriptType);
sb.append(",\n\"timeout\" :").append(timeout);
sb.append('}');
return sb.toString();
}
public void validate(String version) throws SliderException {
Metainfo.checkNonNull(getScript(), "script", "commandScript");
Metainfo.checkNonNull(getScriptType(), "scriptType", "commandScript");
}
}

217
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
View File

@ -1,217 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.SliderException;
import org.codehaus.jackson.annotate.JsonProperty;
import java.util.ArrayList;
import java.util.List;
/**
* Component defined in master package metainfo.json
*/
public class Component extends AbstractComponent {
String category = CATEGORY_MASTER;
String publishConfig = Boolean.FALSE.toString();
String minInstanceCount = "0";
String maxInstanceCount;
String autoStartOnFailure = Boolean.FALSE.toString();
String appExports;
String compExports;
String type = TYPE_STANDARD;
List<ComponentExport> componentExports = new ArrayList<>();
List<DockerContainer> dockerContainers = new ArrayList<>();
List<ConfigFile> configFiles = new ArrayList<>();
public Component() {
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public String getCategory() {
return category;
}
public void setCategory(String category) {
this.category = category;
}
public String getPublishConfig() {
return publishConfig;
}
public void setPublishConfig(String publishConfig) {
this.publishConfig = publishConfig;
}
public String getAutoStartOnFailure() {
return autoStartOnFailure;
}
public void setAutoStartOnFailure(String autoStartOnFailure) {
this.autoStartOnFailure = autoStartOnFailure;
}
public String getAppExports() {
return appExports;
}
public void setAppExports(String appExports) {
this.appExports = appExports;
}
public String getCompExports() {
return compExports;
}
public void setCompExports(String compExports) {
this.compExports = compExports;
}
public String getMinInstanceCount() {
return minInstanceCount;
}
@JsonProperty("dockerContainers")
public List<DockerContainer> getDockerContainers() {
return this.dockerContainers;
}
public Boolean getAutoStartOnFailureBoolean() {
if (SliderUtils.isUnset(getAutoStartOnFailure())) {
return Boolean.FALSE;
}
return Boolean.parseBoolean(getAutoStartOnFailure());
}
public int getMinInstanceCountInt() throws BadConfigException {
if (SliderUtils.isUnset(minInstanceCount)) {
return 0;
}
try {
return Integer.parseInt(minInstanceCount);
} catch (NumberFormatException nfe) {
throw new BadConfigException(nfe, "Invalid value for minInstanceCount for %s", name);
}
}
public int getMaxInstanceCountInt() throws BadConfigException {
if (SliderUtils.isUnset(maxInstanceCount)) {
return Integer.MAX_VALUE;
}
try {
return Integer.parseInt(maxInstanceCount);
} catch (NumberFormatException nfe) {
throw new BadConfigException(nfe, "Invalid value for maxInstanceCount for %s", name);
}
}
public void setMinInstanceCount(String minInstanceCount) {
this.minInstanceCount = minInstanceCount;
}
public String getMaxInstanceCount() {
return maxInstanceCount;
}
public void setMaxInstanceCount(String maxInstanceCount) {
this.maxInstanceCount = maxInstanceCount;
}
public void addComponentExport(ComponentExport export) {
componentExports.add(export);
}
public List<ComponentExport> getComponentExports() {
return componentExports;
}
public Boolean getRequiresAutoRestart() {
return Boolean.parseBoolean(this.autoStartOnFailure);
}
public void addConfigFile(ConfigFile configFile) {
this.configFiles.add(configFile);
}
@JsonProperty("configFiles")
public List<ConfigFile> getConfigFiles() {
return configFiles;
}
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder("{");
sb.append("\n\"name\": ").append(name);
sb.append(",\n\"category\": ").append(category);
sb.append(",\n\"commandScript\" :").append(commandScript);
for(DockerContainer dc : dockerContainers){
sb.append(",\n\"container\" :").append(dc.toString());
}
sb.append('}');
return sb.toString();
}
public void validate(String version) throws SliderException {
Metainfo.checkNonNull(getName(), "name", "component");
Metainfo.checkNonNull(getCategory(), "category", "component");
if (!getCategory().equals(CATEGORY_MASTER)
&& !getCategory().equals(CATEGORY_SLAVE)
&& !getCategory().equals(CATEGORY_CLIENT)) {
throw new SliderException("Invalid category for the component " + getCategory());
}
Metainfo.checkNonNull(getType(), "type", "component");
if (!getType().equals(TYPE_DOCKER)
&& !getType().equals(TYPE_STANDARD)) {
throw new SliderException("Invalid type for the component " + getType());
}
if (version.equals(Metainfo.VERSION_TWO_ZERO)) {
if (getType().equals(TYPE_DOCKER)) {
throw new SliderException(TYPE_DOCKER + " is not supported in version " + Metainfo.VERSION_TWO_ZERO);
}
if (getCommands().size() > 0) {
throw new SliderException("commands are not supported in version " + Metainfo.VERSION_TWO_ZERO);
}
}
if (commandScript != null) {
commandScript.validate(version);
}
if (version.equals(Metainfo.VERSION_TWO_ONE)) {
for (ComponentCommand cc : getCommands()) {
cc.validate(version);
}
}
}
}

85
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java
View File

@ -1,85 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.core.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Represents the metadata associated with the application.
*/
public class ComponentCommand implements Validate {
protected static final Logger
log = LoggerFactory.getLogger(ComponentCommand.class);
private String exec;
private String name = "START";
private String type = "SHELL";
/**
* Creator.
*/
public ComponentCommand() {
}
public void setName(String name) {
this.name = name;
}
public String getName() {
return name;
}
public void setExec(String exec) {
this.exec = exec;
}
public String getExec() {
return exec;
}
public void setType(String type) {
this.type = type;
}
public String getType() {
return type;
}
public void validate(String version) throws SliderException {
Metainfo.checkNonNull(getName(), "name", "componentCommand");
Metainfo.checkNonNull(getType(), "version", "application");
}
public static ComponentCommand getDefaultComponentCommand() {
ComponentCommand cc = new ComponentCommand();
cc.setExec("DEFAULT");
return cc;
}
public static ComponentCommand getDefaultComponentCommand(String commandName) {
ComponentCommand cc = new ComponentCommand();
cc.setExec("DEFAULT");
cc.setName(commandName);
return cc;
}
}

54
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java
View File

@ -1,54 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
/**
*
*/
public class ComponentExport {
String name;
String value;
public ComponentExport() {
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder("{");
sb.append(",\n\"name\": ").append(name);
sb.append(",\n\"value\": ").append(value);
sb.append('}');
return sb.toString();
}
}

26
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java
View File

@ -1,26 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.core.exceptions.SliderException;
public class ComponentsInAddonPackage extends AbstractComponent {
@Override
public void validate(String version) throws SliderException {
}
}

59
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
View File

@ -1,59 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.core.exceptions.SliderException;
/**
*
*/
public class ConfigFile implements Validate {
String type;
String fileName;
String dictionaryName;
public ConfigFile() {
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public String getDictionaryName() {
return dictionaryName;
}
public void setDictionaryName(String dictionaryName) {
this.dictionaryName = dictionaryName;
}
public void validate(String version) throws SliderException {
}
}

39
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java
View File

@ -1,39 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import java.util.ArrayList;
import java.util.List;
/**
* Application default config
*/
public class DefaultConfig {
List<PropertyInfo> propertyInfos;
public DefaultConfig() {
propertyInfos = new ArrayList<PropertyInfo>();
}
public void addPropertyInfo(PropertyInfo propertyInfo) {
propertyInfos.add(propertyInfo);
}
public List<PropertyInfo> getPropertyInfos() {
return propertyInfos;
}
}

54
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java
View File

@ -1,54 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.commons.digester.Digester;
import org.xml.sax.SAXException;
import java.io.IOException;
import java.io.InputStream;
/**
*
*/
public class DefaultConfigParser {
public DefaultConfig parse(InputStream configFileStream) throws IOException {
Digester digester = new Digester();
digester.setValidating(false);
digester.addObjectCreate("configuration", DefaultConfig.class);
digester.addObjectCreate("*/property", PropertyInfo.class);
digester.addBeanPropertySetter("*/property/name");
digester.addBeanPropertySetter("*/property/value");
digester.addBeanPropertySetter("*/property/description");
digester.addSetNext("*/property", "addPropertyInfo");
try {
return (DefaultConfig) digester.parse(configFileStream);
} catch (IOException e) {
} catch (SAXException e) {
} finally {
configFileStream.close();
}
return null;
}
}

187
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java
View File

@ -1,187 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.core.exceptions.SliderException;
import org.codehaus.jackson.annotate.JsonProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
/**
* Represents a docker container
*/
public class DockerContainer implements Validate {
protected static final Logger
log = LoggerFactory.getLogger(DockerContainer.class);
private String name;
private String image;
private String network;
private String useNetworkScript;
private String options;
private List<DockerContainerMount> mounts = new ArrayList<>();
private List<DockerContainerPort> ports = new ArrayList<>();
private String statusCommand;
private String startCommand;
private String commandPath;
private String additionalParam;
private String runPrivilegedContainer;
private List<DockerContainerInputFile> inputFiles = new ArrayList<>();
private List<ConfigFile> configFiles = new ArrayList<>();
public DockerContainer() {
}
@JsonProperty("mounts")
public List<DockerContainerMount> getMounts() { return this.mounts; }
@JsonProperty("ports")
public List<DockerContainerPort> getPorts() {
return this.ports;
}
@JsonProperty("inputFiles")
public List<DockerContainerInputFile> getInputFiles() {
return this.inputFiles;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getImage() {
return image;
}
public void setImage(String image) {
this.image = image;
}
public String getNetwork() {
return network;
}
public void setNetwork(String network) {
this.network = network;
}
public String getUseNetworkScript() {
return useNetworkScript;
}
public void setUseNetworkScript(String useNetworkScript) {
this.useNetworkScript = useNetworkScript;
}
public String getOptions() {
return options;
}
public void setOptions(String options) {
this.options = options;
}
@Override
public void validate(String version) throws SliderException {
Metainfo.checkNonNull(getName(), "name", "dockerContainer");
Metainfo.checkNonNull(getImage(), "image", "dockerContainer");
for (DockerContainerMount dcm : getMounts()) {
dcm.validate(version);
}
for (DockerContainerPort dcp : getPorts()) {
dcp.validate(version);
}
}
@JsonProperty("statusCommand")
public String getStatusCommand() {
return statusCommand;
}
@JsonProperty("statusCommand")
public void setStatusCommand(String statusCommand) {
this.statusCommand = statusCommand;
}
public String getCommandPath() {
return commandPath;
}
public void setCommandPath(String commandPath) {
this.commandPath = commandPath;
}
public String getAdditionalParam() {
return additionalParam;
}
public void setAdditionalParam(String additionalParam) {
this.additionalParam = additionalParam;
}
@JsonProperty("startCommand")
public String getStartCommand() {
return startCommand;
}
@JsonProperty("startCommand")
public void setStartCommand(String startCommand) {
this.startCommand = startCommand;
}
@JsonProperty("runPrivilegedContainer")
public String getRunPrivilegedContainer() {
return runPrivilegedContainer;
}
@JsonProperty("runPrivilegedContainer")
public void setRunPrivilegedContainer(String runPrivilegedContainer) {
this.runPrivilegedContainer = runPrivilegedContainer;
}
public List<ConfigFile> getConfigFiles() {
return configFiles;
}
public void setConfigFiles(List<ConfigFile> configFiles) {
this.configFiles = configFiles;
}
@Override
public String toString() {
StringBuilder result = new StringBuilder("DockerContainer [name=")
.append(name).append(", image=").append(image).append(", options=")
.append(options).append(", mounts=").append(mounts).append(", ports=")
.append(ports).append(", statusCommand=").append(statusCommand)
.append(", commandPath=").append(commandPath)
.append(", additionalParam=").append(additionalParam)
.append(", inputFiles=").append(inputFiles).append(", startCommand=")
.append(startCommand).append(", runPriviledgedContainer=")
.append(runPrivilegedContainer).append(", configFiles=")
.append(configFiles).append("]");
return result.toString();
}
}

50
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java
View File

@ -1,50 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DockerContainerInputFile {
protected static final Logger log = LoggerFactory
.getLogger(DockerContainerInputFile.class);
private String containerPath;
private String fileLocalPath;
public DockerContainerInputFile() {
}
public String getContainerMount() {
return containerPath;
}
public void setContainerMount(String containerMount) {
this.containerPath = containerMount;
}
public String getFileLocalPath() {
return fileLocalPath;
}
public void setFileLocalPath(String fileLocalPath) {
this.fileLocalPath = fileLocalPath;
}
}

60
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java
View File

@ -1,60 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.core.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Represents a docker container mount
*/
public class DockerContainerMount implements Validate {
protected static final Logger
log = LoggerFactory.getLogger(DockerContainerMount.class);
private String containerMount;
private String hostMount;
public DockerContainerMount() {
}
public String getContainerMount() {
return containerMount;
}
public void setContainerMount(String containerMount) {
this.containerMount = containerMount;
}
public String getHostMount() {
return hostMount;
}
public void setHostMount(String hostMount) {
this.hostMount = hostMount;
}
@Override
public void validate(String version) throws SliderException {
Metainfo.checkNonNull(getContainerMount(), "containerMount", "dockerContainerMount");
Metainfo.checkNonNull(getHostMount(), "hostMount", "dockerContainerMount");
}
}

66
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java
View File

@ -1,66 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.core.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Represents a docker container port
*/
public class DockerContainerPort implements Validate {
protected static final Logger
log = LoggerFactory.getLogger(DockerContainerPort.class);
private String containerPort;
private String hostPort;
public DockerContainerPort() {
}
public String getContainerPort() {
return containerPort;
}
public void setContainerPort(String containerPort) {
this.containerPort = containerPort;
}
public String getHostPort() {
return hostPort;
}
public void setHostPort(String hostPort) {
this.hostPort = hostPort;
}
@Override
public void validate(String version) throws SliderException {
Metainfo.checkNonNull(getContainerPort(), "containerPort", "dockerContainerPort");
Metainfo.checkNonNull(getHostPort(), "hostPort", "dockerContainerPort");
}
@Override
public String toString() {
return "DockerContainerPort [containerPort=" + containerPort
+ ", hostPort=" + hostPort + "]";
}
}

61
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java
View File

@ -1,61 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.core.exceptions.SliderException;
/**
*
*/
public class Export implements Validate {
String name;
String value;
public Export() {
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder("{");
sb.append(",\n\"name\": ").append(name);
sb.append(",\n\"value\": ").append(value);
sb.append('}');
return sb.toString();
}
public void validate(String version) throws SliderException {
Metainfo.checkNonNull(getName(), "name", "export");
Metainfo.checkNonNull(getValue(), "value", "export");
}
}

71
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java
View File

@ -1,71 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.core.exceptions.SliderException;
import java.util.ArrayList;
import java.util.List;
/**
*
*/
public class ExportGroup implements Validate {
String name;
List<Export> exports;
public ExportGroup() {
exports = new ArrayList<Export>();
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public void addExport(Export export) {
exports.add(export);
}
public List<Export> getExports() {
return exports;
}
@Override
public String toString() {
final StringBuilder sb =
new StringBuilder("{");
sb.append(",\n\"name\": ").append(name);
sb.append(",\n\"exports\" : {");
for (Export export : exports) {
sb.append("\n").append(export);
}
sb.append("\n},");
sb.append('}');
return sb.toString();
}
public void validate(String version) throws SliderException {
Metainfo.checkNonNull(getName(), "name", "exportGroup");
for(Export exp : getExports()) {
exp.validate(version);
}
}
}

118
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
View File

@ -1,118 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
/**
* Application metainfo uber class
*/
public class Metainfo {
protected static final Logger log =
LoggerFactory.getLogger(Metainfo.class);
public static String VERSION_TWO_ZERO = "2.0";
public static String VERSION_TWO_ONE = "2.1";
String schemaVersion;
ApplicationPackage applicationPackage;
Application application;
public String getSchemaVersion() {
return schemaVersion;
}
public void setSchemaVersion(String schemaVersion) {
this.schemaVersion = schemaVersion;
}
public ApplicationPackage getApplicationPackage() {
return applicationPackage;
}
public void setApplicationPackage(ApplicationPackage pkg) {
this.applicationPackage = pkg;
}
public Application getApplication() {
return application;
}
public void setApplication(Application application) {
this.application = application;
}
public Component getApplicationComponent(String roleGroup) {
if (application == null) {
log.error("Malformed app definition: Expect application as the top level element for metainfo");
} else {
for (Component component : application.getComponents()) {
if (component.getName().equals(roleGroup)) {
return component;
}
}
}
return null;
}
public List<ConfigFile> getComponentConfigFiles(String roleGroup) {
List<ConfigFile> componentConfigFiles = new ArrayList<>();
componentConfigFiles.addAll(application.getConfigFiles());
Component component = getApplicationComponent(roleGroup);
if (component != null) {
componentConfigFiles.addAll(component.getConfigFiles());
}
return componentConfigFiles;
}
public void validate() throws SliderException {
if (!VERSION_TWO_ONE.equals(schemaVersion) &&
!VERSION_TWO_ZERO.equals(schemaVersion)) {
throw new SliderException("Unsupported version " + getSchemaVersion());
}
if (application != null) {
application.validate(schemaVersion);
}
if (applicationPackage != null) {
applicationPackage.validate(schemaVersion);
}
}
public static void checkNonNull(String value, String field, String type) throws SliderException {
if (SliderUtils.isUnset(value)) {
throw new SliderException(type + "." + field + " cannot be null");
}
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("Metainfo [schemaVersion=");
builder.append(schemaVersion);
builder.append(", applicationPackage=");
builder.append(applicationPackage);
builder.append(", application=");
builder.append(application);
builder.append("]");
return builder.toString();
}
}

97
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
View File

@ -1,97 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.commons.digester.Digester;
/**
*
*/
public class MetainfoParser extends AbstractMetainfoParser{
protected void composeSchema(Digester digester){
digester.addObjectCreate("metainfo", Metainfo.class);
digester.addBeanPropertySetter("metainfo/schemaVersion");
digester.addObjectCreate("*/application", Application.class);
digester.addBeanPropertySetter("*/application/name");
digester.addBeanPropertySetter("*/application/comment");
digester.addBeanPropertySetter("*/application/version");
digester.addBeanPropertySetter("*/application/exportedConfigs");
digester.addObjectCreate("*/commandOrder", CommandOrder.class);
digester.addBeanPropertySetter("*/commandOrder/command");
digester.addBeanPropertySetter("*/commandOrder/requires");
digester.addSetNext("*/commandOrder", "addCommandOrder");
digester.addObjectCreate("*/exportGroup", ExportGroup.class);
digester.addBeanPropertySetter("*/exportGroup/name");
digester.addObjectCreate("*/export", Export.class);
digester.addBeanPropertySetter("*/export/name");
digester.addBeanPropertySetter("*/export/value");
digester.addSetNext("*/export", "addExport");
digester.addSetNext("*/exportGroup", "addExportGroup");
digester.addObjectCreate("*/component", Component.class);
digester.addBeanPropertySetter("*/component/name");
digester.addBeanPropertySetter("*/component/category");
digester.addBeanPropertySetter("*/component/publishConfig");
digester.addBeanPropertySetter("*/component/minInstanceCount");
digester.addBeanPropertySetter("*/component/maxInstanceCount");
digester.addBeanPropertySetter("*/component/autoStartOnFailure");
digester.addBeanPropertySetter("*/component/appExports");
digester.addBeanPropertySetter("*/component/compExports");
digester.addObjectCreate("*/componentExport", ComponentExport.class);
digester.addBeanPropertySetter("*/componentExport/name");
digester.addBeanPropertySetter("*/componentExport/value");
digester.addSetNext("*/componentExport", "addComponentExport");
digester.addSetNext("*/component", "addComponent");
digester.addObjectCreate("*/commandScript", CommandScript.class);
digester.addBeanPropertySetter("*/commandScript/script");
digester.addBeanPropertySetter("*/commandScript/scriptType");
digester.addBeanPropertySetter("*/commandScript/timeout");
digester.addSetNext("*/commandScript", "addCommandScript");
digester.addObjectCreate("*/command", ComponentCommand.class);
digester.addBeanPropertySetter("*/command/exec");
digester.addBeanPropertySetter("*/command/name");
digester.addBeanPropertySetter("*/command/type");
digester.addSetNext("*/command", "addCommand");
digester.addObjectCreate("*/osSpecific", OSSpecific.class);
digester.addBeanPropertySetter("*/osSpecific/osType");
digester.addObjectCreate("*/osSpecific/packages/package", OSPackage.class);
digester.addBeanPropertySetter("*/osSpecific/packages/package/type");
digester.addBeanPropertySetter("*/osSpecific/packages/package/name");
digester.addSetNext("*/osSpecific/packages/package", "addOSPackage");
digester.addSetNext("*/osSpecific", "addOSSpecific");
digester.addObjectCreate("*/application/packages/package", Package.class);
digester.addBeanPropertySetter("*/application/packages/package/type");
digester.addBeanPropertySetter("*/application/packages/package/name");
digester.addSetNext("*/application/packages/package", "addPackage");
digester.addObjectCreate("*/configFile", ConfigFile.class);
digester.addBeanPropertySetter("*/configFile/type");
digester.addBeanPropertySetter("*/configFile/fileName");
digester.addBeanPropertySetter("*/configFile/dictionaryName");
digester.addSetNext("*/configFile", "addConfigFile");
digester.addSetRoot("*/application", "setApplication");
}
}

51
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java
View File

@ -1,51 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.core.exceptions.SliderException;
/**
*
*/
public class OSPackage implements Validate {
String type;
String name;
public OSPackage() {
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public void validate(String version) throws SliderException {
Metainfo.checkNonNull(getName(), "name", "osPackage");
Metainfo.checkNonNull(getType(), "type", "osPackage");
}
}

57
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java
View File

@ -1,57 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.core.exceptions.SliderException;
import java.util.ArrayList;
import java.util.List;
/**
*
*/
public class OSSpecific implements Validate {
String osType;
List<OSPackage> packages;
public OSSpecific() {
packages = new ArrayList<OSPackage>();
}
public String getOsType() {
return osType;
}
public void setOsType(String osType) {
this.osType = osType;
}
public void addOSPackage(OSPackage osPackage) {
packages.add(osPackage);
}
public List<OSPackage> getPackages() {
return packages;
}
public void validate(String version) throws SliderException {
Metainfo.checkNonNull(getOsType(), "osType", "osSpecific");
for (OSPackage opkg : getPackages()) {
opkg.validate(version);
}
}
}

60
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java
View File

@ -1,60 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.core.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Represents package description.
*/
public class Package implements Validate {
protected static final Logger
log = LoggerFactory.getLogger(Package.class);
private String name;
private String type;
public Package() {
}
public void setName(String name) {
this.name = name;
}
public String getName() {
return name;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public void validate(String version) throws SliderException {
Metainfo.checkNonNull(getName(), "name", "package");
Metainfo.checkNonNull(getType(), "type", "package");
}
}

54
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java
View File

@ -1,54 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
/**
* Application config property info
*/
public class PropertyInfo {
String name;
String value;
String description;
public PropertyInfo() {
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
}

27
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java
View File

@ -1,27 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.core.exceptions.SliderException;
/**
* Implementer provides a validate method
*/
public interface Validate {
public void validate(String version) throws SliderException;
}

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md
View File

@ -1,22 +0,0 @@
<!---
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
# todo list
* Retry on failure
* Agent can toleate a configurable number of failures (e.g. 3) before giving up
* Agent should separate out hostname and label that is received for registration

138
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
View File

@ -20,12 +20,8 @@ package org.apache.slider.server.appmaster;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.health.HealthCheckRegistry;
import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
import com.google.common.base.Preconditions;
import com.google.protobuf.BlockingService;
import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@ -36,8 +32,14 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@ -66,18 +68,9 @@ import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG;
import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
@ -122,18 +115,16 @@ import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.SliderProviderFactory;
import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.providers.agent.AgentProviderService;
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.providers.slideram.SliderAMProviderService;
import org.apache.slider.server.appmaster.actions.ActionRegisterServiceInstance;
import org.apache.slider.server.appmaster.actions.EscalateOutstandingRequests;
import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
import org.apache.slider.server.appmaster.actions.QueueExecutor;
import org.apache.slider.server.appmaster.actions.QueueService;
import org.apache.slider.server.appmaster.actions.ActionStopSlider;
import org.apache.slider.server.appmaster.actions.ActionUpgradeContainers;
import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.actions.EscalateOutstandingRequests;
import org.apache.slider.server.appmaster.actions.QueueExecutor;
import org.apache.slider.server.appmaster.actions.QueueService;
import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
import org.apache.slider.server.appmaster.actions.RenewingAction;
import org.apache.slider.server.appmaster.actions.ResetFailureWindow;
import org.apache.slider.server.appmaster.actions.ReviewAndFlexApplicationSize;
@ -143,26 +134,24 @@ import org.apache.slider.server.appmaster.management.YarnServiceHealthCheck;
import org.apache.slider.server.appmaster.monkey.ChaosKillAM;
import org.apache.slider.server.appmaster.monkey.ChaosKillContainer;
import org.apache.slider.server.appmaster.monkey.ChaosMonkeyService;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.operations.AsyncRMOperationHandler;
import org.apache.slider.server.appmaster.operations.ProviderNotifyingOperationHandler;
import org.apache.slider.server.appmaster.operations.RMOperationHandler;
import org.apache.slider.server.appmaster.rpc.RpcBinder;
import org.apache.slider.server.appmaster.rpc.SliderAMPolicyProvider;
import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPBImpl;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.rpc.SliderIPCService;
import org.apache.slider.server.appmaster.security.SecurityConfiguration;
import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
import org.apache.slider.server.appmaster.state.ContainerAssignment;
import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.operations.RMOperationHandler;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.web.AgentService;
import org.apache.slider.server.appmaster.web.rest.InsecureAmFilterInitializer;
import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp;
import org.apache.slider.server.appmaster.web.SliderAMWebApp;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.WebAppApiImpl;
import org.apache.slider.server.appmaster.web.rest.InsecureAmFilterInitializer;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.appmaster.web.rest.application.ApplicationResouceContentCacheFactory;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
@ -181,7 +170,6 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.ByteBuffer;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
@ -199,6 +187,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG;
/**
* This is the AM, which directly implements the callbacks from the AM and NM
*/
@ -766,11 +757,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
clustername);
certificateManager.setPassphrase(instanceDefinition.getPassphrase());
if (component.getOptionBool(
AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false)) {
uploadServerCertForLocalization(clustername, fs);
}
// Web service endpoints: initialize
WebAppApiImpl webAppApi =
new WebAppApiImpl(
@ -784,10 +770,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
contentCache);
initAMFilterOptions(serviceConf);
if (providerService instanceof AgentProviderService) {
// start the agent web app
startAgentWebApp(appInformation, serviceConf, webAppApi);
}
int webAppPort = deployWebApplication(webAppApi);
String scheme = WebAppUtils.HTTP_PREFIX;
@ -1165,26 +1147,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
return portScanner.getAvailablePort();
}
private void uploadServerCertForLocalization(String clustername,
SliderFileSystem fs)
throws IOException {
Path certsDir = fs.buildClusterSecurityDirPath(clustername);
if (!fs.getFileSystem().exists(certsDir)) {
fs.getFileSystem().mkdirs(certsDir,
new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
}
Path destPath = new Path(certsDir, SliderKeys.CRT_FILE_NAME);
if (!fs.getFileSystem().exists(destPath)) {
fs.getFileSystem().copyFromLocalFile(
new Path(CertificateManager.getServerCertficateFilePath().getAbsolutePath()),
destPath);
log.info("Uploaded server cert to localization path {}", destPath);
}
fs.getFileSystem().setPermission(destPath,
new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE));
}
protected void login(String principal, File localKeytabFile)
throws IOException, SliderException {
log.info("Logging in as {} with keytab {}", principal, localKeytabFile);
@ -1220,53 +1182,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
}
/**
* Set up and start the agent web application
* @param appInformation application information
* @param serviceConf service configuration
* @param webAppApi web app API instance to bind to
* @throws IOException
*/
private void startAgentWebApp(MapOperations appInformation,
Configuration serviceConf, WebAppApiImpl webAppApi) throws IOException, SliderException {
URL[] urls = ((URLClassLoader) AgentWebApp.class.getClassLoader() ).getURLs();
StringBuilder sb = new StringBuilder("AM classpath:");
for (URL url : urls) {
sb.append("\n").append(url.toString());
}
LOG_YARN.debug(sb.append("\n").toString());
initAMFilterOptions(serviceConf);
// Start up the agent web app and track the URL for it
MapOperations appMasterConfig = getInstanceDefinition()
.getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM);
AgentWebApp agentWebApp = AgentWebApp.$for(AgentWebApp.BASE_PATH,
webAppApi,
RestPaths.AGENT_WS_CONTEXT)
.withComponentConfig(appMasterConfig)
.withPort(getPortToRequest())
.withSecuredPort(getPortToRequest())
.start();
agentOpsUrl =
"https://" + appMasterHostname + ":" + agentWebApp.getSecuredPort();
agentStatusUrl =
"https://" + appMasterHostname + ":" + agentWebApp.getPort();
AgentService agentService =
new AgentService("slider-agent", agentWebApp);
agentService.init(serviceConf);
agentService.start();
addService(agentService);
appInformation.put(StatusKeys.INFO_AM_AGENT_OPS_URL, agentOpsUrl + "/");
appInformation.put(StatusKeys.INFO_AM_AGENT_STATUS_URL, agentStatusUrl + "/");
appInformation.set(StatusKeys.INFO_AM_AGENT_STATUS_PORT,
agentWebApp.getPort());
appInformation.set(StatusKeys.INFO_AM_AGENT_OPS_PORT,
agentWebApp.getSecuredPort());
}
/**
* Set up the AM filter
* @param serviceConf configuration to patch
@ -1327,17 +1242,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
serviceRecord);
// provider service dynamic definitions.
if (providerService instanceof AgentProviderService) {
URL agentOpsURI = new URL(agentOpsUrl);
URL agentStatusURI = new URL(agentStatusUrl);
((AgentProviderService)providerService).applyInitialRegistryDefinitions(
amWebURI,
agentOpsURI,
agentStatusURI,
serviceRecord);
} else {
providerService.applyInitialRegistryDefinitions(amWebURI, serviceRecord);
}
// set any provided attributes
setProvidedServiceRecordAttributes(
@ -1800,11 +1706,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
LOG_YARN.info("Final list of containers to be upgraded (total {}) : {}",
containers.size(), containers);
if (providerService instanceof AgentProviderService) {
AgentProviderService agentProviderService = (AgentProviderService) providerService;
agentProviderService.setInUpgradeMode(true);
agentProviderService.addUpgradeContainers(containers);
}
}
// create a reverse map of roles -> set of all live containers
@ -1968,11 +1869,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* Shutdown operation: release all containers
*/
private void releaseAllContainers() {
if (providerService instanceof AgentProviderService) {
log.info("Setting stopInitiated flag to true");
AgentProviderService agentProviderService = (AgentProviderService) providerService;
agentProviderService.setAppStopInitiated(true);
}
// Add the sleep here (before releasing containers) so that applications get
// time to perform graceful shutdown
try {

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java
View File

@ -1,37 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web;
import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp;
import org.apache.slider.server.services.workflow.ClosingService;
import org.apache.slider.server.services.workflow.WorkflowCompositeService;
/**
* agent service gives the agent webapp lifecycle integration
*/
public class AgentService extends ClosingService<AgentWebApp> {
public AgentService(String name) {
super(name);
}
public AgentService(String name, AgentWebApp app) {
super(name, app);
}
}

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
View File

@ -17,7 +17,6 @@
package org.apache.slider.server.appmaster.web;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.AppMasterActionOperations;
import org.apache.slider.server.appmaster.actions.QueueAccess;
@ -25,7 +24,6 @@ import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
import org.apache.slider.server.services.security.CertificateManager;
@ -58,11 +56,6 @@ public interface WebAppApi {
*/
Map<String, RoleStatus> getRoleStatusByName();
/**
* Returns an interface that can support the agent-based REST operations.
*/
AgentRestOperations getAgentRestOperations();
/**
* Registry operations accessor
* @return registry access

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
View File

@ -23,7 +23,6 @@ import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
import org.apache.slider.server.services.security.CertificateManager;
import org.slf4j.Logger;
@ -96,11 +95,6 @@ public class WebAppApiImpl implements WebAppApi {
return map;
}
@Override
public AgentRestOperations getAgentRestOperations() {
return provider.getAgentRestOperations();
}
@Override
public RegistryOperations getRegistryOperations() {
return registryOperations;

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
View File

@ -19,7 +19,6 @@ package org.apache.slider.server.appmaster.web.rest;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.rest.agent.AgentResource;
import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource;
import org.apache.slider.server.appmaster.web.rest.management.ManagementResource;
import org.apache.slider.server.appmaster.web.rest.publisher.PublisherResource;

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java
View File

@ -1,23 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
public enum AgentCommandType {
EXECUTION_COMMAND,
STATUS_COMMAND,
REGISTRATION_COMMAND
}

376
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java
View File

@ -1,376 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
import com.google.gson.annotations.SerializedName;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class AgentEnv {
/**
* Various directories, configurable in <code>ambari-agent.ini</code>
*/
private Directory[] stackFoldersAndFiles = new Directory[0];
/**
* Directories that match name <code>/etc/alternatives/*conf</code>
*/
private Alternative[] alternatives = new Alternative[0];
/**
* List of existing users
*/
private ExistingUser[] existingUsers = new ExistingUser[0];
/**
* List of repos
*/
private String[] existingRepos = new String[0];
/**
* List of packages
*/
private PackageDetail[] installedPackages = new PackageDetail[0];
/**
* The host health report
*/
private HostHealth hostHealth = new HostHealth();
private Integer umask;
private Boolean iptablesIsRunning;
public Integer getUmask() {
return umask;
}
public void setUmask(Integer umask) {
this.umask = umask;
}
public Directory[] getStackFoldersAndFiles() {
return stackFoldersAndFiles;
}
public void setStackFoldersAndFiles(Directory[] dirs) {
stackFoldersAndFiles = dirs;
}
public void setExistingUsers(ExistingUser[] users) {
existingUsers = users;
}
public ExistingUser[] getExistingUsers() {
return existingUsers;
}
public void setAlternatives(Alternative[] dirs) {
alternatives = dirs;
}
public Alternative[] getAlternatives() {
return alternatives;
}
public void setExistingRepos(String[] repos) {
existingRepos = repos;
}
public String[] getExistingRepos() {
return existingRepos;
}
public void setInstalledPackages(PackageDetail[] packages) {
installedPackages = packages;
}
public PackageDetail[] getInstalledPackages() {
return installedPackages;
}
public void setHostHealth(HostHealth healthReport) {
hostHealth = healthReport;
}
public HostHealth getHostHealth() {
return hostHealth;
}
public Boolean getIptablesIsRunning() {
return iptablesIsRunning;
}
public void setIptablesIsRunning(Boolean iptablesIsRunning) {
this.iptablesIsRunning = iptablesIsRunning;
}
public static class HostHealth {
/**
* Java processes running on the system. Default empty array.
*/
@SerializedName("activeJavaProcs")
private JavaProc[] activeJavaProcs = new JavaProc[0];
/**
* The current time when agent send the host check report
*/
@SerializedName("agentTimeStampAtReporting")
private long agentTimeStampAtReporting = 0;
/**
* The current time when host check report was received
*/
@SerializedName("serverTimeStampAtReporting")
private long serverTimeStampAtReporting = 0;
/**
* Live services running on the agent
*/
@SerializedName("liveServices")
private LiveService[] liveServices = new LiveService[0];
public void setAgentTimeStampAtReporting(long currentTime) {
agentTimeStampAtReporting = currentTime;
}
public long getAgentTimeStampAtReporting() {
return agentTimeStampAtReporting;
}
public void setServerTimeStampAtReporting(long currentTime) {
serverTimeStampAtReporting = currentTime;
}
public long getServerTimeStampAtReporting() {
return serverTimeStampAtReporting;
}
public void setActiveJavaProcs(JavaProc[] procs) {
activeJavaProcs = procs;
}
public JavaProc[] getActiveJavaProcs() {
return activeJavaProcs;
}
public void setLiveServices(LiveService[] services) {
liveServices = services;
}
public LiveService[] getLiveServices() {
return liveServices;
}
}
public static class PackageDetail {
@SerializedName("name")
private String pkgName;
@SerializedName("version")
private String pkgVersion;
@SerializedName("repoName")
private String pkgRepoName;
public void setName(String name) {
pkgName = name;
}
public String getName() {
return pkgName;
}
public void setVersion(String version) {
pkgVersion = version;
}
public String getVersion() {
return pkgVersion;
}
public void setRepoName(String repoName) {
pkgRepoName = repoName;
}
public String getRepoName() {
return pkgRepoName;
}
}
/**
* Represents information about a directory of interest.
*/
public static class Directory {
@SerializedName("name")
private String dirName;
@SerializedName("type")
private String dirType;
public void setName(String name) {
dirName = name;
}
public String getName() {
return dirName;
}
public void setType(String type) {
dirType = type;
}
public String getType() {
return dirType;
}
}
/**
* Represents information about running java processes.
*/
public static class JavaProc {
@SerializedName("user")
private String user;
@SerializedName("pid")
private int pid = 0;
@SerializedName("hadoop")
private boolean is_hadoop = false;
@SerializedName("command")
private String command;
public void setUser(String user) {
this.user = user;
}
public String getUser() {
return user;
}
public void setPid(int pid) {
this.pid = pid;
}
public int getPid() {
return pid;
}
public void setHadoop(boolean hadoop) {
is_hadoop = hadoop;
}
public boolean isHadoop() {
return is_hadoop;
}
public void setCommand(String cmd) {
command = cmd;
}
public String getCommand() {
return command;
}
}
public static class Alternative {
@SerializedName("name")
private String altName;
@SerializedName("target")
private String altTarget;
public void setName(String name) {
altName = name;
}
public String getName() {
return altName;
}
public void setTarget(String target) {
altTarget = target;
}
public String getTarget() {
return altTarget;
}
}
public static class LiveService {
@SerializedName("name")
private String svcName;
@SerializedName("status")
private String svcStatus;
@SerializedName("desc")
private String svcDesc;
public void setName(String name) {
svcName = name;
}
public String getName() {
return svcName;
}
public void setStatus(String status) {
svcStatus = status;
}
public String getStatus() {
return svcStatus;
}
public void setDesc(String desc) {
svcDesc = desc;
}
public String getDesc() {
return svcDesc;
}
}
public static class ExistingUser {
@SerializedName("name")
private String name;
@SerializedName("homeDir")
private String homeDir;
@SerializedName("status")
private String status;
public void setUserName(String userName) {
name = userName;
}
public String getUserName() {
return name;
}
public void setUserHomeDir(String userHomeDir) {
homeDir = userHomeDir;
}
public String getUserHomeDir() {
return homeDir;
}
public void setUserStatus(String userStatus) {
status = userStatus;
}
public String getUserStatus() {
return status;
}
}
}

118
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
View File

@ -1,118 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.services.security.SignCertResponse;
import org.apache.slider.server.services.security.SignMessage;
import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
/**
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class AgentResource extends AbstractSliderResource {
private String agent_name;
public AgentResource(WebAppApi slider) {
super(slider);
}
private void init(HttpServletResponse res) {
res.setContentType(null);
}
@GET
@Path("/agent/register")
public Response endpointAgentRegister() {
Response response = Response.status(200).entity("/agent/register").build();
return response;
}
@GET
@Path("/agent")
public Response endpointAgent() {
Response response = Response.status(200).entity("/agent").build();
return response;
}
@GET
@Path("/")
public Response endpointRoot() {
Response response = Response.status(200).entity("/").build();
return response;
}
@POST
@Path("/{agent_name: [a-zA-Z][a-zA-Z0-9_-]*}/register")
@Consumes({MediaType.APPLICATION_JSON})
@Produces({MediaType.APPLICATION_JSON})
public RegistrationResponse register(Register registration,
@Context HttpServletResponse res,
@PathParam("agent_name") String agent_name) {
init(res);
this.agent_name = agent_name;
AgentRestOperations ops = slider.getAgentRestOperations();
return ops.handleRegistration(registration);
}
@POST
@Path("/{agent_name: [a-zA-Z][a-zA-Z0-9_-]*}/heartbeat")
@Consumes(MediaType.APPLICATION_JSON)
@Produces({MediaType.APPLICATION_JSON})
public HeartBeatResponse heartbeat(HeartBeat message,
@Context HttpServletResponse res,
@PathParam("agent_name") String agent_name) {
init(res);
AgentRestOperations ops = slider.getAgentRestOperations();
return ops.handleHeartBeat(message);
}
@GET
@Path("/cert/ca")
@Produces({MediaType.TEXT_PLAIN})
public String downloadSrvrCrt() {
return slider.getCertificateManager().getServerCert();
}
@Path("/certs/{hostName}")
@POST
@Consumes(MediaType.APPLICATION_JSON)
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
public SignCertResponse signAgentCrt(@PathParam("hostName") String hostname,
SignMessage message, @Context HttpServletRequest req) {
return slider.getCertificateManager().signAgentCrt(hostname,
message.getCsr(),
message.getPassphrase());
}
}

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentRestOperations.java
View File

@ -1,28 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
/**
*
*/
public interface AgentRestOperations {
RegistrationResponse handleRegistration(Register registration);
HeartBeatResponse handleHeartBeat(HeartBeat heartBeat);
}

258
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
View File

@ -1,258 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
import com.google.common.base.Preconditions;
import com.sun.jersey.api.core.ResourceConfig;
import com.sun.jersey.spi.container.WebApplication;
import com.sun.jersey.spi.container.servlet.ServletContainer;
import com.sun.jersey.spi.container.servlet.WebConfig;
import com.sun.jersey.spi.inject.SingletonTypeInjectableProvider;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.services.security.SecurityUtils;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.security.SslSelectChannelConnector;
import org.mortbay.jetty.servlet.Context;
import org.mortbay.jetty.servlet.ServletHolder;
import org.mortbay.thread.QueuedThreadPool;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.ws.rs.ext.Provider;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.net.BindException;
import java.util.Set;
/**
*
*/
public class AgentWebApp implements Closeable {
protected static final Logger LOG = LoggerFactory.getLogger(AgentWebApp.class);
private int port;
private int securedPort;
private static Server agentServer;
public static final String BASE_PATH = "slideragent";
public static class Builder {
final String name;
final String wsName;
final WebAppApi application;
int port;
int securedPort;
MapOperations configsMap;
public Builder(String name, String wsName, WebAppApi application) {
this.name = name;
this.wsName = wsName;
this.application = application;
}
public Builder withComponentConfig(MapOperations appMasterConfig) {
this.configsMap = appMasterConfig;
return this;
}
public Builder withPort (int port) {
this.port = port;
return this;
}
public Builder withSecuredPort (int securedPort) {
this.securedPort = securedPort;
return this;
}
public AgentWebApp start() throws IOException {
if (configsMap == null) {
throw new IllegalStateException("No SSL Configuration Available");
}
agentServer = new Server();
agentServer.setThreadPool(
new QueuedThreadPool(
configsMap.getOptionInt("agent.threadpool.size.max", 25)));
agentServer.setStopAtShutdown(true);
agentServer.setGracefulShutdown(1000);
SslSelectChannelConnector ssl1WayConnector = createSSLConnector(false, port);
SslSelectChannelConnector ssl2WayConnector =
createSSLConnector(Boolean.valueOf(
configsMap.getOption(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED,
"false")), securedPort);
agentServer.setConnectors(new Connector[]{ssl1WayConnector,
ssl2WayConnector});
ServletHolder agent = new ServletHolder(new AgentServletContainer());
Context agentRoot = new Context(agentServer, "/", Context.SESSIONS);
agent.setInitParameter("com.sun.jersey.config.property.resourceConfigClass",
"com.sun.jersey.api.core.PackagesResourceConfig");
agent.setInitParameter("com.sun.jersey.config.property.packages",
"org.apache.slider.server.appmaster.web.rest.agent");
agent.setInitParameter("com.sun.jersey.api.json.POJOMappingFeature",
"true");
// agent.setInitParameter("com.sun.jersey.spi.container.ContainerRequestFilters", "com.sun.jersey.api.container.filter.LoggingFilter");
// agent.setInitParameter("com.sun.jersey.spi.container.ContainerResponseFilters", "com.sun.jersey.api.container.filter.LoggingFilter");
// agent.setInitParameter("com.sun.jersey.config.feature.Trace", "true");
agentRoot.addServlet(agent, "/*");
try {
openListeners();
agentServer.start();
} catch (IOException e) {
LOG.error("Unable to start agent server", e);
throw e;
} catch (Exception e) {
LOG.error("Unable to start agent server", e);
throw new IOException("Unable to start agent server: " + e, e);
}
AgentWebApp webApp = new AgentWebApp();
webApp.setPort(getConnectorPort(agentServer, 0));
webApp.setSecuredPort(getConnectorPort(agentServer, 1));
return webApp;
}
private void openListeners() throws Exception {
// from HttpServer2.openListeners()
for (Connector listener : agentServer.getConnectors()) {
if (listener.getLocalPort() != -1) {
// This listener is either started externally or has been bound
continue;
}
int port = listener.getPort();
while (true) {
// jetty has a bug where you can't reopen a listener that previously
// failed to open w/o issuing a close first, even if the port is changed
try {
listener.close();
listener.open();
LOG.info("Jetty bound to port " + listener.getLocalPort());
break;
} catch (BindException ex) {
if (port == 0) {
BindException be = new BindException("Port in use: "
+ listener.getHost() + ":" + listener.getPort());
be.initCause(ex);
throw be;
}
}
// try the next port number
listener.setPort(++port);
Thread.sleep(100);
}
}
}
private SslSelectChannelConnector createSSLConnector(boolean needClientAuth, int port) {
SslSelectChannelConnector sslConnector = new
SslSelectChannelConnector();
String keystore = SecurityUtils.getSecurityDir() +
File.separator + "keystore.p12";
String srvrCrtPass = SecurityUtils.getKeystorePass();
sslConnector.setKeystore(keystore);
sslConnector.setTruststore(keystore);
sslConnector.setPassword(srvrCrtPass);
sslConnector.setKeyPassword(srvrCrtPass);
sslConnector.setTrustPassword(srvrCrtPass);
sslConnector.setKeystoreType("PKCS12");
sslConnector.setTruststoreType("PKCS12");
sslConnector.setNeedClientAuth(needClientAuth);
sslConnector.setPort(port);
sslConnector.setAcceptors(2);
return sslConnector;
}
@Provider
public class WebAppApiProvider extends
SingletonTypeInjectableProvider<javax.ws.rs.core.Context, WebAppApi> {
public WebAppApiProvider () {
super(WebAppApi.class, application);
}
}
public class AgentServletContainer extends ServletContainer {
public AgentServletContainer() {
super();
}
@Override
protected void configure(WebConfig wc,
ResourceConfig rc,
WebApplication wa) {
super.configure(wc, rc, wa);
Set<Object> singletons = rc.getSingletons();
singletons.add(new WebAppApiProvider());
}
}
private int getConnectorPort(Server webServer, int index) {
Preconditions.checkArgument(index >= 0);
if (index > webServer.getConnectors().length)
throw new IllegalStateException("Illegal connect index requested");
Connector c = webServer.getConnectors()[index];
if (c.getLocalPort() == -1) {
// The connector is not bounded
throw new IllegalStateException("The connector is not bound to a port");
}
return c.getLocalPort();
}
}
public static Builder $for(String name, WebAppApi app, String wsPrefix) {
return new Builder(name, wsPrefix, app);
}
public int getPort() {
return port;
}
public void setPort(int port) {
this.port = port;
}
public void setSecuredPort(int securedPort) {
this.securedPort = securedPort;
}
public int getSecuredPort() {
return securedPort;
}
public void close() throws IOException{
//need to stop server and reset injector
try {
agentServer.stop();
} catch (IOException e) {
throw e;
} catch (Exception e) {
throw new IOException(e.toString(), e);
}
}
}

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebServices.java
View File

@ -1,40 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import javax.ws.rs.Path;
import javax.ws.rs.core.Context;
/** The available agent REST services exposed by a slider AM. */
@Path(RestPaths.SLIDER_AGENT_CONTEXT_ROOT)
public class AgentWebServices {
/** AM/WebApp info object */
@Context
private WebAppApi slider;
public AgentWebServices() {
}
@Path(RestPaths.SLIDER_SUBPATH_AGENTS)
public AgentResource getAgentResource () {
return new AgentResource(slider);
}
}

207
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java
View File

@ -1,207 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import java.util.Map;
/**
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class CommandReport {
int exitCode;
private String role;
private String actionId;
private String stdout;
private String stderr;
private String structuredOut;
private String status;
private String clusterName;
private String serviceName;
private long taskId;
private String roleCommand;
private Map<String, String> folders;
private Map<String, String> allocatedPorts;
private Map<String, Map<String, String>> configurationTags;
@JsonProperty("taskId")
public long getTaskId() {
return taskId;
}
@JsonProperty("taskId")
public void setTaskId(long taskId) {
this.taskId = taskId;
}
@JsonProperty("clusterName")
public String getClusterName() {
return this.clusterName;
}
@JsonProperty("clusterName")
public void setClusterName(String clusterName) {
this.clusterName = clusterName;
}
@JsonProperty("actionId")
public String getActionId() {
return this.actionId;
}
@JsonProperty("actionId")
public void setActionId(String actionId) {
this.actionId = actionId;
}
@JsonProperty("stderr")
public String getStdErr() {
return this.stderr;
}
@JsonProperty("stderr")
public void setStdErr(String stderr) {
this.stderr = stderr;
}
@JsonProperty("exitcode")
public int getExitCode() {
return this.exitCode;
}
@JsonProperty("exitcode")
public void setExitCode(int exitCode) {
this.exitCode = exitCode;
}
@JsonProperty("stdout")
public String getStdOut() {
return this.stdout;
}
@JsonProperty("stdout")
public void setStdOut(String stdout) {
this.stdout = stdout;
}
@JsonProperty("structuredOut")
public String getStructuredOut() {
return this.structuredOut;
}
@JsonProperty("structuredOut")
public void setStructuredOut(String structuredOut) {
this.structuredOut = structuredOut;
}
@JsonProperty("roleCommand")
public String getRoleCommand() {
return this.roleCommand;
}
@JsonProperty("roleCommand")
public void setRoleCommand(String roleCommand) {
this.roleCommand = roleCommand;
}
@JsonProperty("role")
public String getRole() {
return role;
}
@JsonProperty("role")
public void setRole(String role) {
this.role = role;
}
@JsonProperty("status")
public String getStatus() {
return status;
}
@JsonProperty("status")
public void setStatus(String status) {
this.status = status;
}
@JsonProperty("serviceName")
public String getServiceName() {
return serviceName;
}
@JsonProperty("serviceName")
public void setServiceName(String serviceName) {
this.serviceName = serviceName;
}
/** @return the config tags that match this command, or <code>null</code> if none are present */
@JsonProperty("configurationTags")
public Map<String, Map<String, String>> getConfigurationTags() {
return configurationTags;
}
/** @param tags the config tags that match this command */
@JsonProperty("configurationTags")
public void setConfigurationTags(Map<String, Map<String, String>> tags) {
configurationTags = tags;
}
/** @return the allocated ports, or <code>null</code> if none are present */
@JsonProperty("allocatedPorts")
public Map<String, String> getAllocatedPorts() {
return allocatedPorts;
}
/** @param ports allocated ports */
@JsonProperty("allocatedPorts")
public void setAllocatedPorts(Map<String, String> ports) {
this.allocatedPorts = ports;
}
/** @return the folders, or <code>null</code> if none are present */
@JsonProperty("folders")
public Map<String, String> getFolders() {
return folders;
}
/** @param folders allocated ports */
@JsonProperty("folders")
public void setFolders(Map<String, String> folders) {
this.folders = folders;
}
@Override
public String toString() {
return "CommandReport{" +
"role='" + role + '\'' +
", actionId='" + actionId + '\'' +
", status='" + status + '\'' +
", exitCode=" + exitCode +
", clusterName='" + clusterName + '\'' +
", serviceName='" + serviceName + '\'' +
", taskId=" + taskId +
", roleCommand=" + roleCommand +
", configurationTags=" + configurationTags +
'}';
}
}

129
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java
View File

@ -1,129 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import java.util.Map;
/**
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class ComponentStatus {
String componentName;
String msg;
String status;
String serviceName;
String clusterName;
String roleCommand;
String ip;
String hostname;
@JsonProperty("configurations")
private Map<String, Map<String, String>> configurations;
public String getRoleCommand() {
return roleCommand;
}
public void setRoleCommand(String roleCommand) {
this.roleCommand = roleCommand;
}
public String getComponentName() {
return this.componentName;
}
public void setComponentName(String componentName) {
this.componentName = componentName;
}
public String getMessage() {
return this.msg;
}
public void setMessage(String msg) {
this.msg = msg;
}
public String getStatus() {
return this.status;
}
public void setStatus(String status) {
this.status = status;
}
public String getServiceName() {
return serviceName;
}
public void setServiceName(String serviceName) {
this.serviceName = serviceName;
}
public String getClusterName() {
return clusterName;
}
public void setClusterName(String clusterName) {
this.clusterName = clusterName;
}
/** @return the config tags that match this command, or <code>null</code> if none are present */
public Map<String, Map<String, String>> getConfigs() {
return configurations;
}
/** @param configs the config tags that match this status */
public void setConfigs(Map<String, Map<String, String>> configs) {
this.configurations = configs;
}
@Override
public String toString() {
return "ComponentStatus{" +
"componentName='" + componentName + '\'' +
", msg='" + msg + '\'' +
", status='" + status + '\'' +
", serviceName='" + serviceName + '\'' +
", clusterName='" + clusterName + '\'' +
", roleCommand='" + roleCommand + '\'' +
", ip='" + ip + '\'' +
", hostname='" + hostname + '\'' +
'}';
}
public String getIp() {
return ip;
}
public void setIp(String ip) {
this.ip = ip;
}
public String getHostname() {
return hostname;
}
public void setHostname(String hostname) {
this.hostname = hostname;
}
}

128
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/DiskInfo.java
View File

@ -1,128 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.annotate.JsonSerialize;
/**
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class DiskInfo {
String available;
String mountpoint;
String device;
String used;
String percent;
String size;
String type;
/**
* DiskInfo object that tracks information about a disk.
* @param mountpoint
* @param available
* @param used
* @param percent
* @param size
*/
public DiskInfo(String device, String mountpoint, String available,
String used, String percent, String size, String type) {
this.device = device;
this.mountpoint = mountpoint;
this.available = available;
this.used = used;
this.percent = percent;
this.size = size;
this.type = type;
}
/**
* Needed for Serialization
*/
public DiskInfo() {}
@JsonProperty("available")
public void setAvailable(String available) {
this.available = available;
}
@JsonProperty("available")
public String getAvailable() {
return this.available;
}
@JsonProperty("mountpoint")
public String getMountPoint() {
return this.mountpoint;
}
@JsonProperty("mountpoint")
public void setMountPoint(String mountpoint) {
this.mountpoint = mountpoint;
}
@JsonProperty("type")
public String getType() {
return this.type;
}
@JsonProperty("type")
public void setType(String type) {
this.type = type;
}
@JsonProperty("used")
public String getUsed() {
return this.used;
}
@JsonProperty("used")
public void setUsed(String used) {
this.used = used;
}
@JsonProperty("percent")
public String getPercent() {
return this.percent;
}
@JsonProperty("percent")
public void setPercent(String percent) {
this.percent = percent;
}
@JsonProperty("size")
public String getSize() {
return this.size;
}
@JsonProperty("size")
public void setSize(String size) {
this.size = size;
}
@Override
public String toString() {
return "available=" + this.available + ",mountpoint=" + this.mountpoint
+ ",used=" + this.used + ",percent=" + this.percent + ",size=" +
this.size + ",device=" + this.device +
",type=" + this.type;
}
}

310
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
View File

@ -1,310 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.providers.agent.application.metadata.DockerContainer;
import org.apache.slider.providers.agent.application.metadata.DockerContainerInputFile;
import org.apache.slider.providers.agent.application.metadata.DockerContainerMount;
import org.apache.slider.providers.agent.application.metadata.DockerContainerPort;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class ExecutionCommand {
protected static final Logger log =
LoggerFactory.getLogger(ExecutionCommand.class);
private AgentCommandType commandType = AgentCommandType.EXECUTION_COMMAND;
private String clusterName;
private long taskId;
private String commandId;
//TODO Remove hostname from being set in the command
private String hostname;
private String role;
private Map<String, String> hostLevelParams = new HashMap<String, String>();
private Map<String, String> roleParams = null;
private String roleCommand;
private Map<String, Map<String, String>> configurations;
private Map<String, Map<String, String>> componentConfigurations;
private Map<String, String> commandParams;
private String serviceName;
private String componentName;
private String componentType;
private List<DockerContainer> containers = new ArrayList<>();
private String pkg;
private boolean yarnDockerMode = false;
public ExecutionCommand(AgentCommandType commandType) {
this.commandType = commandType;
}
@JsonProperty("commandType")
public AgentCommandType getCommandType() {
return commandType;
}
@JsonProperty("commandType")
public void setCommandType(AgentCommandType commandType) {
this.commandType = commandType;
}
@JsonProperty("commandId")
public String getCommandId() {
return this.commandId;
}
@JsonProperty("commandId")
public void setCommandId(String commandId) {
this.commandId = commandId;
}
@JsonProperty("taskId")
public long getTaskId() {
return taskId;
}
@JsonProperty("taskId")
public void setTaskId(long taskId) {
this.taskId = taskId;
}
@JsonProperty("role")
public String getRole() {
return role;
}
@JsonProperty("role")
public void setRole(String role) {
this.role = role;
}
@JsonProperty("roleParams")
public Map<String, String> getRoleParams() {
return roleParams;
}
@JsonProperty("roleParams")
public void setRoleParams(Map<String, String> roleParams) {
this.roleParams = roleParams;
}
@JsonProperty("roleCommand")
public String getRoleCommand() {
return roleCommand;
}
@JsonProperty("roleCommand")
public void setRoleCommand(String cmd) {
this.roleCommand = cmd;
}
@JsonProperty("clusterName")
public String getClusterName() {
return clusterName;
}
@JsonProperty("clusterName")
public void setClusterName(String clusterName) {
this.clusterName = clusterName;
}
@JsonProperty("componentType")
public String getComponentType() {
return componentType;
}
@JsonProperty("componentType")
public void setComponentType(String componentType) {
this.componentType = componentType;
}
@JsonProperty("hostname")
public String getHostname() {
return hostname;
}
@JsonProperty("hostname")
public void setHostname(String hostname) {
this.hostname = hostname;
}
@JsonProperty("hostLevelParams")
public Map<String, String> getHostLevelParams() {
return hostLevelParams;
}
@JsonProperty("hostLevelParams")
public void setHostLevelParams(Map<String, String> params) {
this.hostLevelParams = params;
}
@JsonProperty("configurations")
public Map<String, Map<String, String>> getConfigurations() {
return configurations;
}
@JsonProperty("configurations")
public void setConfigurations(Map<String, Map<String, String>> configurations) {
this.configurations = configurations;
}
@JsonProperty("commandParams")
public Map<String, String> getCommandParams() {
return commandParams;
}
@JsonProperty("commandParams")
public void setCommandParams(Map<String, String> commandParams) {
this.commandParams = commandParams;
}
@JsonProperty("serviceName")
public String getServiceName() {
return serviceName;
}
@JsonProperty("serviceName")
public void setServiceName(String serviceName) {
this.serviceName = serviceName;
}
@JsonProperty("componentName")
public String getComponentName() {
return componentName;
}
@JsonProperty("componentName")
public void setComponentName(String componentName) {
this.componentName = componentName;
}
@JsonProperty("package")
public String getPkg() {
return pkg;
}
@JsonProperty("package")
public void setPkg(String pkg) {
this.pkg = pkg;
}
@JsonProperty("componentConfig")
public Map<String, Map<String, String>> getComponentConfigurations() {
return componentConfigurations;
}
@JsonProperty("componentConfig")
public void setComponentConfigurations(
Map<String, Map<String, String>> componentConfigurations) {
this.componentConfigurations = componentConfigurations;
}
@JsonProperty("containers")
public List<DockerContainer> getContainers() {
return containers;
}
@JsonProperty("yarnDockerMode")
public boolean isYarnDockerMode() {
return yarnDockerMode ;
}
@JsonProperty("yarnDockerMode")
public void setYarnDockerMode(boolean yarnDockerMode) {
this.yarnDockerMode = yarnDockerMode;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("ExecutionCommand [commandType=").append(commandType)
.append(", clusterName=").append(clusterName).append(", taskId=")
.append(taskId).append(", commandId=").append(commandId)
.append(", hostname=").append(hostname).append(", role=").append(role)
.append(", hostLevelParams=").append(hostLevelParams)
.append(", roleParams=").append(roleParams).append(", roleCommand=")
.append(roleCommand).append(", configurations=").append(configurations)
.append(", commandParams=").append(commandParams)
.append(", serviceName=").append(serviceName)
.append(", componentName=").append(componentName)
.append(", componentType=").append(componentType)
.append(", yarnDockerMode=").append(yarnDockerMode).append(", pkg=")
.append(pkg).append("]");
return builder.toString();
}
public void addContainerDetails(String componentGroup, Metainfo metaInfo) {
Component component = metaInfo.getApplicationComponent(componentGroup);
this.setComponentType(component.getType());
log.info("Adding container details for {}", componentGroup, " from ",
metaInfo.toString());
for (DockerContainer metaContainer : component.getDockerContainers()) {
DockerContainer container = new DockerContainer();
container.setImage(metaContainer.getImage());
container.setNetwork(metaContainer.getNetwork());
container.setUseNetworkScript(metaContainer.getUseNetworkScript());
container.setName(metaContainer.getName());
container.setOptions(metaContainer.getOptions());
container.setAdditionalParam(metaContainer.getAdditionalParam());
container.setCommandPath(metaContainer.getAdditionalParam());
container.setStatusCommand(metaContainer.getStatusCommand());
container.setStartCommand(metaContainer.getStartCommand());
if (metaContainer.getMounts().size() > 0) {
for (DockerContainerMount metaContMount : metaContainer.getMounts()) {
DockerContainerMount contMnt = new DockerContainerMount();
contMnt.setContainerMount(metaContMount.getContainerMount());
contMnt.setHostMount(metaContMount.getHostMount());
container.getMounts().add(contMnt);
}
}
if (metaContainer.getPorts().size() > 0) {
for (DockerContainerPort metaCntPort : metaContainer.getPorts()) {
DockerContainerPort cntPort = new DockerContainerPort();
cntPort.setContainerPort(metaCntPort.getContainerPort());
cntPort.setHostPort(metaCntPort.getHostPort());
container.getPorts().add(cntPort);
}
}
if (metaContainer.getInputFiles().size() > 0) {
for (DockerContainerInputFile metaInpFile : metaContainer
.getInputFiles()) {
DockerContainerInputFile inpFile = new DockerContainerInputFile();
inpFile.setContainerMount(metaInpFile.getContainerMount());
inpFile.setFileLocalPath(metaInpFile.getFileLocalPath());
container.getInputFiles().add(inpFile);
}
}
if (metaContainer.getConfigFiles() != null) {
container.setConfigFiles(metaContainer.getConfigFiles());
}
log.info("Docker container meta info ready: " + container.toString());
this.getContainers().add(container);
}
}
}

149
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
View File

@ -1,149 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import java.util.ArrayList;
import java.util.List;
/**
*
*
* Data model for agent heartbeat for server (ambari or app master).
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class HeartBeat {
private long responseId = -1;
private long timestamp;
private String hostname;
List<CommandReport> reports = new ArrayList<CommandReport>();
List<ComponentStatus> componentStatus = new ArrayList<ComponentStatus>();
private List<DiskInfo> mounts = new ArrayList<DiskInfo>();
HostStatus nodeStatus;
private AgentEnv agentEnv = null;
private String fqdn;
private String pkg;
public long getResponseId() {
return responseId;
}
public void setResponseId(long responseId) {
this.responseId=responseId;
}
public long getTimestamp() {
return timestamp;
}
public void setTimestamp(long timestamp) {
this.timestamp = timestamp;
}
public String getHostname() {
return hostname;
}
public void setHostname(String hostname) {
this.hostname = hostname;
}
public String getFqdn() {
return fqdn;
}
public void setFqdn(String fqdn) {
this.fqdn = fqdn;
}
@JsonProperty("reports")
public List<CommandReport> getReports() {
return this.reports;
}
@JsonProperty("reports")
public void setReports(List<CommandReport> reports) {
this.reports = reports;
}
public HostStatus getNodeStatus() {
return nodeStatus;
}
public void setNodeStatus(HostStatus nodeStatus) {
this.nodeStatus = nodeStatus;
}
public AgentEnv getAgentEnv() {
return agentEnv;
}
public void setAgentEnv(AgentEnv env) {
agentEnv = env;
}
@JsonProperty("componentStatus")
public List<ComponentStatus> getComponentStatus() {
return componentStatus;
}
@JsonProperty("componentStatus")
public void setComponentStatus(List<ComponentStatus> componentStatus) {
this.componentStatus = componentStatus;
}
@JsonProperty("mounts")
public List<DiskInfo> getMounts() {
return this.mounts;
}
@JsonProperty("mounts")
public void setMounts(List<DiskInfo> mounts) {
this.mounts = mounts;
}
@JsonProperty("package")
public String getPackage() {
return pkg;
}
@JsonProperty("package")
public void setPackage(String pkg) {
this.pkg = pkg;
}
@Override
public String toString() {
return "HeartBeat{" +
"responseId=" + responseId +
", timestamp=" + timestamp +
", hostname='" + hostname + '\'' +
", reports=" + reports +
", componentStatus=" + componentStatus +
", package=" + pkg +
", nodeStatus=" + nodeStatus +
'}';
}
}

147
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
View File

@ -1,147 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import java.util.ArrayList;
import java.util.List;
/**
*
* Controller to Agent response data model.
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class HeartBeatResponse {
private long responseId;
List<ExecutionCommand> executionCommands = new ArrayList<ExecutionCommand>();
List<StatusCommand> statusCommands = new ArrayList<StatusCommand>();
RegistrationCommand registrationCommand;
boolean yarnDockerMode = false;
boolean restartAgent = false;
boolean restartEnabled = true;
boolean hasMappedComponents = false;
boolean terminateAgent = false;
@JsonProperty("responseId")
public long getResponseId() {
return responseId;
}
@JsonProperty("responseId")
public void setResponseId(long responseId) {
this.responseId=responseId;
}
@JsonProperty("executionCommands")
public List<ExecutionCommand> getExecutionCommands() {
return executionCommands;
}
@JsonProperty("executionCommands")
public void setExecutionCommands(List<ExecutionCommand> executionCommands) {
this.executionCommands = executionCommands;
}
@JsonProperty("statusCommands")
public List<StatusCommand> getStatusCommands() {
return statusCommands;
}
@JsonProperty("statusCommands")
public void setStatusCommands(List<StatusCommand> statusCommands) {
this.statusCommands = statusCommands;
}
@JsonProperty("registrationCommand")
public RegistrationCommand getRegistrationCommand() {
return registrationCommand;
}
@JsonProperty("registrationCommand")
public void setRegistrationCommand(RegistrationCommand registrationCommand) {
this.registrationCommand = registrationCommand;
}
@JsonProperty("restartAgent")
public boolean isRestartAgent() {
return restartAgent;
}
@JsonProperty("restartAgent")
public void setRestartAgent(boolean restartAgent) {
this.restartAgent = restartAgent;
}
@JsonProperty("restartEnabled")
public boolean getRstartEnabled() {
return restartEnabled;
}
@JsonProperty("restartEnabled")
public void setRestartEnabled(boolean restartEnabled) {
this.restartEnabled = restartEnabled;
}
@JsonProperty("hasMappedComponents")
public boolean hasMappedComponents() {
return hasMappedComponents;
}
@JsonProperty("hasMappedComponents")
public void setHasMappedComponents(boolean hasMappedComponents) {
this.hasMappedComponents = hasMappedComponents;
}
@JsonProperty("terminateAgent")
public boolean isTerminateAgent() {
return terminateAgent;
}
@JsonProperty("terminateAgent")
public void setTerminateAgent(boolean terminateAgent) {
this.terminateAgent = terminateAgent;
}
public void addExecutionCommand(ExecutionCommand execCmd) {
executionCommands.add(execCmd);
}
public void addStatusCommand(StatusCommand statCmd) {
statusCommands.add(statCmd);
}
@Override
public String toString() {
return "HeartBeatResponse{" +
"responseId=" + responseId +
", executionCommands=" + executionCommands +
", statusCommands=" + statusCommands +
", registrationCommand=" + registrationCommand +
", restartAgent=" + restartAgent +
", terminateAgent=" + terminateAgent +
'}';
}
}

398
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostInfo.java
View File

@ -1,398 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import java.util.ArrayList;
import java.util.List;
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class HostInfo {
private String architecture;
private String domain;
private String fqdn;
private String hardwareisa;
private String hardwaremodel;
private String hostname;
private String id;
private String interfaces;
private String ipaddress;
private String kernel;
private String kernelmajversion;
private String kernelrelease;
private String kernelversion;
private String macaddress;
private long memoryfree;
private long memorysize;
private List<DiskInfo> mounts = new ArrayList<DiskInfo>();
private long memorytotal;
private String netmask;
private String operatingsystem;
private String operatingsystemrelease;
private String osfamily;
private int physicalprocessorcount;
private int processorcount;
private boolean selinux;
private String swapfree;
private String swapsize;
private String timezone;
private String uptime;
private long uptime_days;
private long uptime_hours;
@JsonProperty("architecture")
public String getArchitecture() {
return this.architecture;
}
@JsonProperty("architecture")
public void setArchitecture(String architecture) {
this.architecture = architecture;
}
@JsonProperty("domain")
public String getDomain() {
return this.domain;
}
@JsonProperty("domain")
public void setDomain(String domain) {
this.domain = domain;
}
@JsonProperty("fqdn")
public String getFQDN() {
return this.fqdn;
}
@JsonProperty("fqdn")
public void setFQDN(String fqdn) {
this.fqdn = fqdn;
}
@JsonProperty("hardwareisa")
public String getHardwareIsa() {
return hardwareisa;
}
@JsonProperty("hardwareisa")
public void setHardwareIsa(String hardwareisa) {
this.hardwareisa = hardwareisa;
}
@JsonProperty("hardwaremodel")
public String getHardwareModel() {
return this.hardwaremodel;
}
@JsonProperty("hardwaremodel")
public void setHardwareModel(String hardwaremodel) {
this.hardwaremodel = hardwaremodel;
}
@JsonProperty("hostname")
public String getHostName() {
return this.hostname;
}
@JsonProperty("hostname")
public void setHostName(String hostname) {
this.hostname = hostname;
}
@JsonProperty("id")
public String getAgentUserId() {
return id;
}
@JsonProperty("id")
public void setAgentUserId(String id) {
this.id = id;
}
@JsonProperty("interfaces")
public String getInterfaces() {
return this.interfaces;
}
@JsonProperty("interfaces")
public void setInterfaces(String interfaces) {
this.interfaces = interfaces;
}
@JsonProperty("ipaddress")
public String getIPAddress() {
return this.ipaddress;
}
@JsonProperty("ipaddress")
public void setIPAddress(String ipaddress) {
this.ipaddress = ipaddress;
}
@JsonProperty("kernel")
public String getKernel() {
return this.kernel;
}
@JsonProperty("kernel")
public void setKernel(String kernel) {
this.kernel = kernel;
}
@JsonProperty("kernelmajversion")
public String getKernelMajVersion() {
return this.kernelmajversion;
}
@JsonProperty("kernelmajversion")
public void setKernelMajVersion(String kernelmajversion) {
this.kernelmajversion = kernelmajversion;
}
@JsonProperty("kernelrelease")
public String getKernelRelease() {
return this.kernelrelease;
}
@JsonProperty("kernelrelease")
public void setKernelRelease(String kernelrelease) {
this.kernelrelease = kernelrelease;
}
@JsonProperty("kernelversion")
public String getKernelVersion() {
return this.kernelversion;
}
@JsonProperty("kernelversion")
public void setKernelVersion(String kernelversion) {
this.kernelversion = kernelversion;
}
@JsonProperty("macaddress")
public String getMacAddress() {
return this.macaddress;
}
@JsonProperty("macaddress")
public void setMacAddress(String macaddress) {
this.macaddress = macaddress;
}
@JsonProperty("memoryfree")
public long getFreeMemory() {
return this.memoryfree;
}
@JsonProperty("memoryfree")
public void setFreeMemory(long memoryfree) {
this.memoryfree = memoryfree;
}
@JsonProperty("memorysize")
public long getMemorySize() {
return this.memorysize;
}
@JsonProperty("memorysize")
public void setMemorySize(long memorysize) {
this.memorysize = memorysize;
}
@JsonProperty("mounts")
public List<DiskInfo> getMounts() {
return this.mounts;
}
@JsonProperty("mounts")
public void setMounts(List<DiskInfo> mounts) {
this.mounts = mounts;
}
@JsonProperty("memorytotal")
public long getMemoryTotal() {
return this.memorytotal;
}
@JsonProperty("memorytotal")
public void setMemoryTotal(long memorytotal) {
this.memorytotal = memorytotal;
}
@JsonProperty("netmask")
public String getNetMask() {
return this.netmask;
}
@JsonProperty("netmask")
public void setNetMask(String netmask) {
this.netmask = netmask;
}
@JsonProperty("operatingsystem")
public String getOS() {
return this.operatingsystem;
}
@JsonProperty("operatingsystem")
public void setOS(String operatingsystem) {
this.operatingsystem = operatingsystem;
}
@JsonProperty("operatingsystemrelease")
public String getOSRelease() {
return this.operatingsystemrelease;
}
@JsonProperty("operatingsystemrelease")
public void setOSRelease(String operatingsystemrelease) {
this.operatingsystemrelease = operatingsystemrelease;
}
@JsonProperty("osfamily")
public String getOSFamily() {
return this.osfamily;
}
@JsonProperty("osfamily")
public void setOSFamily(String osfamily) {
this.osfamily = osfamily;
}
@JsonProperty("physicalprocessorcount")
public int getPhysicalProcessorCount() {
return this.physicalprocessorcount;
}
@JsonProperty("physicalprocessorcount")
public void setPhysicalProcessorCount(int physicalprocessorcount) {
this.physicalprocessorcount = physicalprocessorcount;
}
@JsonProperty("processorcount")
public int getProcessorCount() {
return this.processorcount;
}
@JsonProperty("processorcount")
public void setProcessorCount(int processorcount) {
this.processorcount = processorcount;
}
@JsonProperty("selinux")
public boolean getSeLinux() {
return selinux;
}
@JsonProperty("selinux")
public void setSeLinux(boolean selinux) {
this.selinux = selinux;
}
@JsonProperty("swapfree")
public String getSwapFree() {
return this.swapfree;
}
@JsonProperty("swapfree")
public void setSwapFree(String swapfree) {
this.swapfree = swapfree;
}
@JsonProperty("swapsize")
public String getSwapSize() {
return swapsize;
}
@JsonProperty("swapsize")
public void setSwapSize(String swapsize) {
this.swapsize = swapsize;
}
@JsonProperty("timezone")
public String getTimeZone() {
return this.timezone;
}
@JsonProperty("timezone")
public void setTimeZone(String timezone) {
this.timezone = timezone;
}
@JsonProperty("uptime")
public String getUptime() {
return this.uptime;
}
@JsonProperty("uptime")
public void setUpTime(String uptime) {
this.uptime = uptime;
}
@JsonProperty("uptime_hours")
public long getUptimeHours() {
return this.uptime_hours;
}
@JsonProperty("uptime_hours")
public void setUpTimeHours(long uptime_hours) {
this.uptime_hours = uptime_hours;
}
@JsonProperty("uptime_days")
public long getUpTimeDays() {
return this.uptime_days;
}
@JsonProperty("uptime_days")
public void setUpTimeDays(long uptime_days) {
this.uptime_days = uptime_days;
}
private String getDiskString() {
if (mounts == null) {
return null;
}
StringBuilder ret = new StringBuilder();
for (DiskInfo diskInfo : mounts) {
ret.append("(").append(diskInfo.toString()).append(")");
}
return ret.toString();
}
public String toString() {
return "[" +
"hostname=" + this.hostname + "," +
"fqdn=" + this.fqdn + "," +
"domain=" + this.domain + "," +
"architecture=" + this.architecture + "," +
"processorcount=" + this.processorcount + "," +
"physicalprocessorcount=" + this.physicalprocessorcount + "," +
"osname=" + this.operatingsystem + "," +
"osversion=" + this.operatingsystemrelease + "," +
"osfamily=" + this.osfamily + "," +
"memory=" + this.memorytotal + "," +
"uptime_hours=" + this.uptime_hours + "," +
"mounts=" + getDiskString() + "]\n";
}
}

63
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostStatus.java
View File

@ -1,63 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
/**
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class HostStatus {
public HostStatus(Status status, String cause) {
super();
this.status = status;
this.cause = cause;
}
public HostStatus() {
super();
}
public enum Status {
HEALTHY,
UNHEALTHY
}
Status status;
String cause;
public Status getStatus() {
return status;
}
public void setStatus(Status status) {
this.status = status;
}
public String getCause() {
return cause;
}
public void setCause(String cause) {
this.cause = cause;
}
@Override
public String toString() {
return "HostStatus{" +
"status=" + status +
", cause='" + cause + '\'' +
'}';
}
}

193
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
View File

@ -1,193 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
import org.apache.slider.providers.agent.State;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import java.util.Map;
/** Data model for agent to send heartbeat to ambari and/or app master. */
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class Register {
private int responseId = -1;
private long timestamp;
private String label;
private int currentPingPort;
private HostInfo hardwareProfile;
private String publicHostname;
private String tags;
private AgentEnv agentEnv;
private String agentVersion;
private State actualState;
private State expectedState;
private Map<String, String> allocatedPorts;
private Map<String, String> logFolders;
private String pkg;
private String appVersion;
@JsonProperty("responseId")
public int getResponseId() {
return responseId;
}
@JsonProperty("responseId")
public void setResponseId(int responseId) {
this.responseId = responseId;
}
public long getTimestamp() {
return timestamp;
}
public void setTimestamp(long timestamp) {
this.timestamp = timestamp;
}
public String getLabel() {
return label;
}
public void setLabel(String label) {
this.label = label;
}
public String getTags() {
return tags;
}
public void setTags(String tags) {
this.tags = tags;
}
public HostInfo getHardwareProfile() {
return hardwareProfile;
}
public void setHardwareProfile(HostInfo hardwareProfile) {
this.hardwareProfile = hardwareProfile;
}
public String getPublicHostname() {
return publicHostname;
}
public void setPublicHostname(String name) {
this.publicHostname = name;
}
public AgentEnv getAgentEnv() {
return agentEnv;
}
public void setAgentEnv(AgentEnv env) {
this.agentEnv = env;
}
public String getAgentVersion() {
return agentVersion;
}
public void setAgentVersion(String agentVersion) {
this.agentVersion = agentVersion;
}
public int getCurrentPingPort() {
return currentPingPort;
}
public void setCurrentPingPort(int currentPingPort) {
this.currentPingPort = currentPingPort;
}
public State getActualState() {
return actualState;
}
public void setActualState(State actualState) {
this.actualState = actualState;
}
public State getExpectedState() {
return expectedState;
}
public void setExpectedState(State expectedState) {
this.expectedState = expectedState;
}
/** @return the allocated ports, or <code>null</code> if none are present */
@JsonProperty("allocatedPorts")
public Map<String, String> getAllocatedPorts() {
return allocatedPorts;
}
/** @param ports allocated ports */
@JsonProperty("allocatedPorts")
public void setAllocatedPorts(Map<String, String> ports) {
this.allocatedPorts = ports;
}
/** @return the log folders, or <code>null</code> if none are present */
@JsonProperty("logFolders")
public Map<String, String> getLogFolders() {
return logFolders;
}
/** @param logFolders assigned log folders */
@JsonProperty("logFolders")
public void setLogFolders(Map<String, String> logFolders) {
this.logFolders = logFolders;
}
public String getPkg() {
return pkg;
}
public void setPkg(String pkg) {
this.pkg = pkg;
}
@JsonProperty("appVersion")
public String getAppVersion() {
return appVersion;
}
@JsonProperty("appVersion")
public void setAppVersion(String appVersion) {
this.appVersion = appVersion;
}
@Override
public String toString() {
String ret = "responseId=" + responseId + "\n" +
"timestamp=" + timestamp + "\n" +
"label=" + label + "\n" +
"hostname=" + publicHostname + "\n" +
"expectedState=" + expectedState + "\n" +
"actualState=" + actualState + "\n" +
"appVersion=" + appVersion + "\n";
if (hardwareProfile != null) {
ret = ret + "hardwareprofile=" + this.hardwareProfile.toString();
}
return ret;
}
}

43
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationCommand.java
View File

@ -1,43 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
/**
*
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class RegistrationCommand {
private String command;
public String getCommand() {
return command;
}
public void setCommand(String command) {
this.command = command;
}
public RegistrationCommand(String command) {
this.command = command;
}
}

133
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
View File

@ -1,133 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import java.util.List;
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class RegistrationResponse {
@JsonProperty("response")
private RegistrationStatus response;
/**
* exitstatus is a code of error which was rised on server side. exitstatus
* = 0 (OK - Default) exitstatus = 1 (Registration failed because different
* version of agent and server)
*/
@JsonProperty("exitstatus")
private int exitstatus;
/** log - message, which will be printed to agents log */
@JsonProperty("log")
private String log;
/** tags - tags associated with the container */
@JsonProperty("tags")
private String tags;
@JsonProperty("package")
private String pkg;
//Response id to start with, usually zero.
@JsonProperty("responseId")
private long responseId;
@JsonProperty("statusCommands")
private List<StatusCommand> statusCommands = null;
public RegistrationResponse() {
}
public RegistrationStatus getResponse() {
return response;
}
public void setResponse(RegistrationStatus response) {
this.response = response;
}
public int getExitstatus() {
return exitstatus;
}
public void setExitstatus(int exitstatus) {
this.exitstatus = exitstatus;
}
public RegistrationStatus getResponseStatus() {
return response;
}
public void setResponseStatus(RegistrationStatus response) {
this.response = response;
}
public List<StatusCommand> getStatusCommands() {
return statusCommands;
}
public void setStatusCommands(List<StatusCommand> statusCommands) {
this.statusCommands = statusCommands;
}
public long getResponseId() {
return responseId;
}
public void setResponseId(long responseId) {
this.responseId = responseId;
}
public String getTags() {
return tags;
}
public void setTags(String tags) {
this.tags = tags;
}
public String getLog() {
return log;
}
public void setLog(String log) {
this.log = log;
}
public String getPkg() {
return pkg;
}
public void setPkg(String pkg) {
this.pkg = pkg;
}
@Override
public String toString() {
return "RegistrationResponse{" +
"response=" + response +
", responseId=" + responseId +
", statusCommands=" + statusCommands +
'}';
}
}

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java
View File

@ -1,22 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
public enum RegistrationStatus {
OK,
FAILED
}

152
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
View File

@ -1,152 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.agent;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import java.util.HashMap;
import java.util.Map;
/**
* Command to report the status of a list of services in roles.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class StatusCommand {
public static String STATUS_COMMAND = "STATUS";
public static String GET_CONFIG_COMMAND = "GET_CONFIG";
AgentCommandType agentCommandType;
private String clusterName;
private String serviceName;
private String componentName;
private Map<String, Map<String, String>> configurations;
private Map<String, String> commandParams = new HashMap<String, String>();
private Map<String, String> hostLevelParams = new HashMap<String, String>();
private String roleCommand;
private boolean yarnDockerMode;
public StatusCommand() {
this.agentCommandType = AgentCommandType.STATUS_COMMAND;
}
@JsonProperty("clusterName")
public String getClusterName() {
return clusterName;
}
@JsonProperty("clusterName")
public void setClusterName(String clusterName) {
this.clusterName = clusterName;
}
@JsonProperty("serviceName")
public String getServiceName() {
return serviceName;
}
@JsonProperty("serviceName")
public void setServiceName(String serviceName) {
this.serviceName = serviceName;
}
@JsonProperty("componentName")
public String getComponentName() {
return componentName;
}
@JsonProperty("componentName")
public void setComponentName(String componentName) {
this.componentName = componentName;
}
@JsonProperty("configurations")
public Map<String, Map<String, String>> getConfigurations() {
return configurations;
}
@JsonProperty("configurations")
public void setConfigurations(Map<String, Map<String, String>> configurations) {
this.configurations = configurations;
}
@JsonProperty("hostLevelParams")
public Map<String, String> getHostLevelParams() {
return hostLevelParams;
}
@JsonProperty("hostLevelParams")
public void setHostLevelParams(Map<String, String> params) {
this.hostLevelParams = params;
}
@JsonProperty("commandParams")
public Map<String, String> getCommandParams() {
return commandParams;
}
@JsonProperty("commandParams")
public void setCommandParams(Map<String, String> commandParams) {
this.commandParams = commandParams;
}
@JsonProperty("commandType")
public AgentCommandType getCommandType() {
return agentCommandType;
}
@JsonProperty("commandType")
public void setCommandType(AgentCommandType commandType) {
this.agentCommandType = commandType;
}
@JsonProperty("roleCommand")
public String getRoleCommand() {
return roleCommand;
}
@JsonProperty("roleCommand")
public void setRoleCommand(String roleCommand) {
this.roleCommand = roleCommand;
}
@JsonProperty("yarnDockerMode")
public boolean isYarnDockerMode() {
return yarnDockerMode;
}
@JsonProperty("yarnDockerMode")
public void setYarnDockerMode(boolean yarnDockerMode) {
this.yarnDockerMode = yarnDockerMode;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("StatusCommand [agentCommandType=").append(agentCommandType)
.append(", clusterName=").append(clusterName).append(", serviceName=")
.append(serviceName).append(", componentName=").append(componentName)
.append(", configurations=").append(configurations)
.append(", commandParams=").append(commandParams)
.append(", hostLevelParams=").append(hostLevelParams)
.append(", roleCommand=").append(roleCommand).append("]");
return builder.toString();
}
}

77
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
View File

@ -1,77 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.tools.TestUtility;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Set;
/**
*
*/
public class TestAgentClientProvider {
protected static final Logger log =
LoggerFactory.getLogger(TestAgentClientProvider.class);
@Rule
public TemporaryFolder folder = new TemporaryFolder();
@Test
public void testGetApplicationTags() throws Exception {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.getLocal(configuration);
SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
AgentClientProvider provider = new AgentClientProvider(null);
String zipFileName = TestUtility.createAppPackage(
folder,
"testpkg",
"test.zip",
"target/test-classes/org/apache/slider/common/tools/test");
Set<String> tags = provider.getApplicationTags(sliderFileSystem, zipFileName);
assert tags != null;
assert !tags.isEmpty();
assert tags.contains("Name: STORM");
assert tags.contains("Description: Apache Hadoop Stream processing framework");
assert tags.contains("Version: 0.9.1.2.1");
}
@Test
public void testValidateInstanceDefinition() throws Exception {
AgentClientProvider provider = new AgentClientProvider(null);
AggregateConf instanceDefinition = new AggregateConf();
try {
provider.validateInstanceDefinition(instanceDefinition, null);
Assert.assertFalse("Should fail with BadConfigException", true);
} catch (BadConfigException e) {
log.info(e.toString());
Assert.assertTrue(e.getMessage().contains("Application definition must be provided"));
}
}
}

76
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java
View File

@ -1,76 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
*/
public class TestAgentLaunchParameter {
protected static final Logger log =
LoggerFactory.getLogger(TestAgentLaunchParameter.class);
@Test
public void testTestAgentLaunchParameter() throws Exception {
AgentLaunchParameter alp = new AgentLaunchParameter("");
Assert.assertEquals("", alp.getNextLaunchParameter("abc"));
Assert.assertEquals("", alp.getNextLaunchParameter("HBASE_MASTER"));
alp = new AgentLaunchParameter("a:1:2:3|b:5:6:NONE");
Assert.assertEquals("1", alp.getNextLaunchParameter("a"));
Assert.assertEquals("2", alp.getNextLaunchParameter("a"));
Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
Assert.assertEquals("5", alp.getNextLaunchParameter("b"));
Assert.assertEquals("6", alp.getNextLaunchParameter("b"));
Assert.assertEquals("", alp.getNextLaunchParameter("b"));
Assert.assertEquals("", alp.getNextLaunchParameter("b"));
Assert.assertEquals("", alp.getNextLaunchParameter("c"));
alp = new AgentLaunchParameter("|a:1:3|b::5:NONE:");
Assert.assertEquals("1", alp.getNextLaunchParameter("a"));
Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
Assert.assertEquals("", alp.getNextLaunchParameter("b"));
Assert.assertEquals("5", alp.getNextLaunchParameter("b"));
Assert.assertEquals("", alp.getNextLaunchParameter("b"));
Assert.assertEquals("", alp.getNextLaunchParameter("b"));
alp = new AgentLaunchParameter("|:");
Assert.assertEquals("", alp.getNextLaunchParameter("b"));
Assert.assertEquals("", alp.getNextLaunchParameter("a"));
alp = new AgentLaunchParameter("HBASE_MASTER:a,b:DO_NOT_REGISTER:");
Assert.assertEquals("a,b", alp.getNextLaunchParameter("HBASE_MASTER"));
Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER"));
Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER"));
alp = new AgentLaunchParameter("HBASE_MASTER:a,b:DO_NOT_REGISTER::c:::");
Assert.assertEquals("a,b", alp.getNextLaunchParameter("HBASE_MASTER"));
Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER"));
Assert.assertEquals("", alp.getNextLaunchParameter("HBASE_MASTER"));
Assert.assertEquals("c", alp.getNextLaunchParameter("HBASE_MASTER"));
Assert.assertEquals("c", alp.getNextLaunchParameter("HBASE_MASTER"));
}
}

94
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java
View File

@ -1,94 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.tools.TestUtility;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TestAgentUtils {
protected static final Logger log =
LoggerFactory.getLogger(TestAgentUtils.class);
@Rule
public TemporaryFolder folder = new TemporaryFolder();
private static final String metainfo_str = "<metainfo>\n"
+ " <schemaVersion>2.0</schemaVersion>\n"
+ " <application>\n"
+ " <name>MYTESTAPPLICATION</name>\n"
+ " <comment>\n"
+ " My Test Application\n"
+ " </comment>\n"
+ " <version>1.0</version>\n"
+ " <type>YARN-APP</type>\n"
+ " <components>\n"
+ " <component>\n"
+ " <name>REST</name>\n"
+ " <category>MASTER</category>\n"
+ " <commandScript>\n"
+ " <script>scripts/rest.py</script>\n"
+ " <scriptType>PYTHON</scriptType>\n"
+ " <timeout>600</timeout>\n"
+ " </commandScript>\n"
+ " </component>\n"
+ " </components>\n"
+ " </application>\n"
+ "</metainfo>";
@Test
public void testGetApplicationMetainfo() throws Exception {
String zipFileName = TestUtility.createAppPackage(
folder,
"testpkg",
"test.zip",
"target/test-classes/org/apache/slider/common/tools/test");
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.getLocal(configuration);
log.info("fs working dir is {}", fs.getWorkingDirectory().toString());
SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
// Without accompany metainfo file, read metainfo from the zip file
Metainfo metainfo = AgentUtils.getApplicationMetainfo(
sliderFileSystem, zipFileName, false);
Assert.assertNotNull(metainfo.getApplication());
Assert.assertEquals("STORM", metainfo.getApplication().getName());
// With accompany metainfo file, read metainfo from the accompany file
String acompanyFileName = zipFileName + ".metainfo.xml";
File f = new File(acompanyFileName);
try (BufferedWriter writer = new BufferedWriter(new FileWriter(f))) {
writer.write(metainfo_str);
}
metainfo = AgentUtils.getApplicationMetainfo(
sliderFileSystem, zipFileName, false);
Assert.assertNotNull(metainfo.getApplication());
Assert.assertEquals("MYTESTAPPLICATION", metainfo.getApplication().getName());
}
}

264
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
View File

@ -1,264 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
import com.google.common.io.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.slider.common.params.ActionCreateArgs;
import org.apache.slider.common.params.AddonArgsDelegate;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.persist.AppDefinitionPersister;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;
/**
*
*/
public class TestAppDefinitionPersister {
protected static final Logger log =
LoggerFactory.getLogger(TestAppDefinitionPersister.class);
@Rule
public TemporaryFolder folder = new TemporaryFolder();
/**
* @BeforeClass public static void initialize() { BasicConfigurator.resetConfiguration();
* BasicConfigurator.configure(); }*
*/
@Test
public void testAppDefinitionPersister() throws Exception {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.getLocal(configuration);
log.info("fs working dir is {}", fs.getWorkingDirectory().toString());
SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
AppDefinitionPersister adp = new AppDefinitionPersister(sliderFileSystem);
String clustername = "c1";
ActionCreateArgs buildInfo = new ActionCreateArgs();
buildInfo.appMetaInfo = null;
buildInfo.appDef = null;
buildInfo.addonDelegate = new AddonArgsDelegate();
// nothing to do
adp.processSuppliedDefinitions(clustername, buildInfo, null);
adp.persistPackages();
List<AppDefinitionPersister.AppDefinition> appDefinitions = adp.getAppDefinitions();
Assert.assertTrue(appDefinitions.size() == 0);
ConfTree ct = new ConfTree();
ConfTreeOperations appConf = new ConfTreeOperations(ct);
final File tempDir = Files.createTempDir();
final File metainfo = new File(tempDir, "metainfo.json");
// unreadable metainfo
buildInfo.appMetaInfo = metainfo;
try {
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
} catch (BadConfigException bce) {
log.info(bce.getMessage());
Assert.assertTrue(bce.getMessage().contains(
"Path specified with "
+ "--metainfo either cannot be read or is not a file"));
}
try (PrintWriter writer = new PrintWriter(metainfo.getAbsolutePath(), "UTF-8")) {
writer.println("{");
writer.println("}");
}
buildInfo.appDef = metainfo;
try {
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
} catch (BadConfigException bce) {
log.info(bce.getMessage());
Assert.assertTrue(bce.getMessage().contains(
"Both --metainfo and --appdef cannot be specified"));
}
// both --metainfojson and --appdef cannot be specified
buildInfo.appMetaInfo = null;
buildInfo.appMetaInfoJson = "{}";
try {
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
} catch (BadConfigException bce) {
log.info(bce.getMessage());
Assert.assertTrue(bce.getMessage().contains(
"Both --metainfojson and --appdef cannot be specified"));
}
buildInfo.appDef = null;
buildInfo.appMetaInfoJson = "";
try {
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
} catch (BadConfigException bce) {
log.info(bce.getMessage());
Assert.assertTrue(bce.getMessage().contains(
"Empty string specified with --metainfojson"));
}
buildInfo.appMetaInfo = metainfo;
// both --metainfo and --metainfojson cannot be specified
buildInfo.appMetaInfoJson = "{}";
try {
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
} catch (BadConfigException bce) {
log.info(bce.getMessage());
Assert.assertTrue(bce.getMessage().contains(
"Both --metainfo and --metainfojson cannot be specified"));
}
buildInfo.appMetaInfoJson = null;
appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath());
try {
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
} catch (BadConfigException bce) {
log.info(bce.getMessage());
Assert.assertTrue(bce.getMessage().contains(
"application.def cannot "
+ "not be set if --metainfo is specified in the cmd line"));
}
appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
appDefinitions = adp.getAppDefinitions();
Assert.assertTrue(appDefinitions.size() == 1);
Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip"));
log.info(appDefinitions.get(0).toString());
Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith("default"));
Assert.assertTrue(appDefinitions.get(0).targetFolderInFs.toString().contains("cluster/c1/appdef"));
Assert.assertEquals("appPkg.zip", appDefinitions.get(0).pkgName);
buildInfo.appDef = tempDir;
buildInfo.appMetaInfo = null;
appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath());
try {
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
} catch (BadConfigException bce) {
log.info(bce.getMessage());
Assert.assertTrue(bce.getMessage().contains("application.def must not be set if --appdef is provided"));
}
adp.getAppDefinitions().clear();
appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
appDefinitions = adp.getAppDefinitions();
Assert.assertTrue(appDefinitions.size() == 1);
Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip"));
log.info(appDefinitions.get(0).toString());
Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith(tempDir.toString()));
Assert.assertTrue(appDefinitions.get(0).targetFolderInFs.toString().contains("cluster/c1/appdef"));
Assert.assertEquals("appPkg.zip", appDefinitions.get(0).pkgName);
adp.getAppDefinitions().clear();
buildInfo.appDef = null;
buildInfo.appMetaInfo = null;
appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
ArrayList<String> list = new ArrayList<String>() {{
add("addon1");
add("");
add("addon2");
add(metainfo.getAbsolutePath());
}};
buildInfo.addonDelegate.addonTuples = list;
try {
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
} catch (BadConfigException bce) {
log.info(bce.getMessage());
Assert.assertTrue(bce.getMessage().contains("addon package can only be specified if main app package is specified"));
}
buildInfo.appMetaInfo = metainfo;
try {
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
} catch (BadConfigException bce) {
log.info(bce.getMessage());
Assert.assertTrue(bce.getMessage().contains("Invalid path for addon package addon1"));
}
appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
list = new ArrayList<String>() {{
add("addon1");
add(tempDir.getAbsolutePath());
add("addon2");
add(metainfo.getAbsolutePath());
}};
buildInfo.addonDelegate.addonTuples = list;
adp.getAppDefinitions().clear();
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
appDefinitions = adp.getAppDefinitions();
Assert.assertTrue(appDefinitions.size() == 3);
Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip"));
Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon1").contains(
"addons/addon1/addon_addon1.zip"));
Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon2").contains(
"addons/addon2/addon_addon2.zip"));
log.info(appConf.getGlobalOptions().get("application.addons"));
Assert.assertTrue(appConf.getGlobalOptions().get("application.addons").contains(
"application.addon.addon2,application.addon.addon1")
|| appConf.getGlobalOptions().get("application.addons").contains(
"application.addon.addon1,application.addon.addon2"));
int seen = 0;
for (AppDefinitionPersister.AppDefinition adp_ad : appDefinitions) {
if (adp_ad.pkgName.equals("appPkg.zip")) {
log.info(adp_ad.toString());
Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith("default"));
Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("cluster/c1/appdef"));
seen++;
}
if (adp_ad.pkgName.equals("addon_addon1.zip")) {
log.info(adp_ad.toString());
Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith(tempDir.toString()));
Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("addons/addon1"));
seen++;
}
if (adp_ad.pkgName.equals("addon_addon2.zip")) {
log.info(adp_ad.toString());
Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith("metainfo.json"));
Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("addons/addon2"));
seen++;
}
}
Assert.assertEquals(3, seen);
}
}

115
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java
View File

@ -1,115 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TestComponentTagProvider {
protected static final Logger log =
LoggerFactory.getLogger(TestComponentTagProvider.class);
@Test
public void testTagProvider() throws Exception {
ComponentTagProvider ctp = new ComponentTagProvider();
Assert.assertEquals("", ctp.getTag(null, null));
Assert.assertEquals("", ctp.getTag(null, "cid"));
Assert.assertEquals("", ctp.getTag("comp1", null));
Assert.assertEquals("1", ctp.getTag("comp1", "cid1"));
Assert.assertEquals("2", ctp.getTag("comp1", "cid2"));
Assert.assertEquals("3", ctp.getTag("comp1", "cid3"));
ctp.releaseTag("comp1", "cid2");
Assert.assertEquals("2", ctp.getTag("comp1", "cid22"));
ctp.releaseTag("comp1", "cid4");
ctp.recordAssignedTag("comp1", "cid5", "5");
Assert.assertEquals("4", ctp.getTag("comp1", "cid4"));
Assert.assertEquals("4", ctp.getTag("comp1", "cid4"));
Assert.assertEquals("6", ctp.getTag("comp1", "cid6"));
ctp.recordAssignedTag("comp1", "cid55", "5");
Assert.assertEquals("5", ctp.getTag("comp1", "cid55"));
ctp.recordAssignedTag("comp2", "cidb3", "3");
Assert.assertEquals("1", ctp.getTag("comp2", "cidb1"));
Assert.assertEquals("2", ctp.getTag("comp2", "cidb2"));
Assert.assertEquals("4", ctp.getTag("comp2", "cidb4"));
ctp.recordAssignedTag("comp2", "cidb5", "six");
ctp.recordAssignedTag("comp2", "cidb5", "-55");
ctp.recordAssignedTag("comp2", "cidb5", "tags");
ctp.recordAssignedTag("comp2", "cidb5", null);
ctp.recordAssignedTag("comp2", "cidb5", "");
ctp.recordAssignedTag("comp2", "cidb5", "5");
Assert.assertEquals("6", ctp.getTag("comp2", "cidb6"));
ctp.recordAssignedTag("comp2", null, "5");
ctp.recordAssignedTag(null, null, "5");
ctp.releaseTag("comp1", null);
ctp.releaseTag(null, "cid4");
ctp.releaseTag(null, null);
}
@Test
public void testTagProviderWithThread() throws Exception {
ComponentTagProvider ctp = new ComponentTagProvider();
Thread thread = new Thread(new Taggged(ctp));
Thread thread2 = new Thread(new Taggged(ctp));
Thread thread3 = new Thread(new Taggged(ctp));
thread.start();
thread2.start();
thread3.start();
ctp.getTag("comp1", "cid50");
thread.join();
thread2.join();
thread3.join();
Assert.assertEquals("101", ctp.getTag("comp1", "cid101"));
}
public class Taggged implements Runnable {
private final ComponentTagProvider ctp;
public Taggged(ComponentTagProvider ctp) {
this.ctp = ctp;
}
public void run() {
for (int i = 0; i < 100; i++) {
String containerId = "cid" + (i + 1);
this.ctp.getTag("comp1", containerId);
}
for (int i = 0; i < 100; i++) {
String containerId = "cid" + (i + 1);
this.ctp.getTag("comp1", containerId);
}
for (int i = 0; i < 100; i += 2) {
String containerId = "cid" + (i + 1);
this.ctp.releaseTag("comp1", containerId);
}
for (int i = 0; i < 100; i += 2) {
String containerId = "cid" + (i + 1);
this.ctp.getTag("comp1", containerId);
}
}
}
}

33
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java
View File

@ -1,33 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TestState {
protected static final Logger log = LoggerFactory.getLogger(TestState.class);
@Test
public void testState() throws Exception {
State state = State.STARTED;
Assert.assertEquals(Command.STOP, state.getSupportedCommand(false, true));
}
}

107
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java
View File

@ -1,107 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
/**
*
*/
public class TestConfigParser {
protected static final Logger log =
LoggerFactory.getLogger(TestConfigParser.class);
private static final String config_1_str = "<configuration>\n"
+ " <property>\n"
+ " <name>security.client.protocol.acl</name>\n"
+ " <value>*</value>\n"
+ " <description>ACL for HRegionInterface protocol implementations (ie. \n"
+ " clients talking to HRegionServers)\n"
+ " The ACL is a comma-separated list of user and group names. The user and \n"
+ " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n"
+ " A special value of \"*\" means all users are allowed.</description>\n"
+ " </property>\n"
+ "\n"
+ " <property>\n"
+ " <name>security.admin.protocol.acl</name>\n"
+ " <value>*</value>\n"
+ " <description>ACL for HMasterInterface protocol implementation (ie. \n"
+ " clients talking to HMaster for admin operations).\n"
+ " The ACL is a comma-separated list of user and group names. The user and \n"
+ " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n"
+ " A special value of \"*\" means all users are allowed.</description>\n"
+ " </property>\n"
+ "\n"
+ " <property>\n"
+ " <name>security.masterregion.protocol.acl</name>\n"
+ " <value>*</value>\n"
+ " <description>ACL for HMasterRegionInterface protocol implementations\n"
+ " (for HRegionServers communicating with HMaster)\n"
+ " The ACL is a comma-separated list of user and group names. The user and \n"
+ " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n"
+ " A special value of \"*\" means all users are allowed.</description>\n"
+ " </property>\n"
+ " <property>\n"
+ " <name>emptyVal</name>\n"
+ " <value></value>\n"
+ " <description>non-empty-desc</description>\n"
+ " </property>\n"
+ " <property>\n"
+ " <name>emptyDesc</name>\n"
+ " <value></value>\n"
+ " <description></description>\n"
+ " </property>\n"
+ " <property>\n"
+ " <name>noDesc</name>\n"
+ " <value></value>\n"
+ " </property>\n"
+ "</configuration>";
@Test
public void testParse() throws IOException {
InputStream config_1 = new ByteArrayInputStream(config_1_str.getBytes());
DefaultConfig config = new DefaultConfigParser().parse(config_1);
Assert.assertNotNull(config);
Assert.assertNotNull(config.getPropertyInfos());
Assert.assertEquals(6, config.getPropertyInfos().size());
for (PropertyInfo pInfo : config.getPropertyInfos()) {
if (pInfo.getName().equals("security.client.protocol.acl")) {
Assert.assertEquals("*", pInfo.getValue());
Assert.assertTrue(pInfo.getDescription().startsWith("ACL for HRegionInterface "));
}
if (pInfo.getName().equals("emptyVal")) {
Assert.assertEquals("", pInfo.getValue());
Assert.assertEquals("non-empty-desc", pInfo.getDescription());
}
if (pInfo.getName().equals("emptyDesc")) {
Assert.assertEquals("", pInfo.getValue());
Assert.assertEquals("", pInfo.getDescription());
}
if (pInfo.getName().equals("noDesc")) {
Assert.assertEquals("", pInfo.getValue());
Assert.assertNull(pInfo.getDescription());
}
}
}
}

177
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
View File

@ -1,177 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.providers.agent.AgentProviderService;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import static org.mockito.Mockito.doReturn;
/**
*
*/
public class TestMetainfoParser {
protected static final Logger log =
LoggerFactory.getLogger(TestMetainfoParser.class);
public static final String METAINFO_XML =
"/org/apache/slider/providers/agent/application/metadata/metainfo.xml";
@Test
public void testParse() throws IOException {
InputStream resStream = this.getClass().getResourceAsStream(
METAINFO_XML);
MetainfoParser parser = new MetainfoParser();
Metainfo metainfo = parser.fromXmlStream(resStream);
Assert.assertNotNull(metainfo);
Assert.assertNotNull(metainfo.getApplication());
Application application = metainfo.getApplication();
assert "STORM".equals(application.getName());
assert 6 == application.getComponents().size();
OSPackage pkg = application.getOSSpecifics().get(0).getPackages().get(0);
assert "tarball".equals(pkg.getType());
assert "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz".equals(pkg.getName());
boolean found = false;
for (Component comp : application.getComponents()) {
if (comp != null && comp.getName().equals("NIMBUS")) {
found = true;
Assert.assertEquals(0, comp.getComponentExports().size());
}
if (comp != null && comp.getName().equals("SUPERVISOR")) {
Assert.assertEquals(1, comp.getComponentExports().size());
}
if (comp != null && comp.getName().equals("ANOTHER_COMPONENT")) {
assert 2 == comp.getCommands().size();
assert "start command".equals(comp.getCommands().get(0).getExec());
assert "START".equals(comp.getCommands().get(0).getName());
assert "stop command".equals(comp.getCommands().get(1).getExec());
assert "STOP".equals(comp.getCommands().get(1).getName());
}
}
assert found;
Assert.assertEquals(0, application.getConfigFiles().size());
assert 1 == application.getPackages().size();
Package p = application.getPackages().get(0);
assert "tarball".equals(p.getType());
assert "test-tarball-name.tgz".equals(p.getName());
}
@Test
public void testJsonParse() throws IOException {
String metaInfo1_json = "{\n"
+ "\"schemaVersion\":\"2.2\",\n"
+ "\"application\":{\n"
+ "\"name\": \"MEMCACHED\","
+ "\"exportGroups\": ["
+ "{"
+ "\"name\": \"Servers\","
+ "\"exports\": ["
+ "{"
+ "\"name\": \"host_port\","
+ "\"value\": \"${MEMCACHED_HOST}:${site.global.port}\""
+ "}"
+ "]"
+ "}"
+ "],"
+ "\"components\": ["
+ "{"
+ "\"name\": \"MEMCACHED\","
+ "\"compExports\": \"Servers-host_port\","
+ "\"commands\": ["
+ "{"
+ "\"exec\": \"java -classpath /usr/myapps/memcached/*:/usr/lib/hadoop/lib/* com.thimbleware.jmemcached.Main\""
+ "}"
+ "]"
+ "},"
+ "{"
+ "\"name\": \"MEMCACHED2\","
+ "\"commands\": ["
+ "{"
+ "\"exec\": \"scripts/config.py\","
+ "\"type\": \"PYTHON\","
+ "\"name\": \"CONFIGURE\""
+ "}"
+ "],"
+ "\"dockerContainers\": ["
+ "{"
+ "\"name\": \"redis\","
+ "\"image\": \"dockerhub/redis\","
+ "\"options\": \"-net=bridge\","
+ "\"mounts\": ["
+ "{"
+ "\"containerMount\": \"/tmp/conf\","
+ "\"hostMount\": \"{$conf:@//site/global/app_root}/conf\""
+ "}"
+ "]"
+ "}"
+ "]"
+ "}"
+ "]"
+ "}"
+ "}";
MetainfoParser parser = new MetainfoParser();
Metainfo mInfo = parser.fromJsonString(metaInfo1_json);
Assert.assertEquals("2.2", mInfo.getSchemaVersion());
Application app = mInfo.getApplication();
Assert.assertNotNull(app);
Assert.assertEquals("MEMCACHED", app.getName());
List<ExportGroup> egs = app.getExportGroups();
Assert.assertEquals(1, egs.size());
ExportGroup eg = egs.get(0);
Assert.assertEquals("Servers", eg.getName());
List<Export> exports = eg.getExports();
Assert.assertEquals(1, exports.size());
Export export = exports.get(0);
Assert.assertEquals("host_port", export.getName());
Assert.assertEquals("${MEMCACHED_HOST}:${site.global.port}", export.getValue());
List<Component> components = app.getComponents();
Assert.assertEquals(2, components.size());
Component c1 = mInfo.getApplicationComponent("MEMCACHED");
Assert.assertNotNull(c1);
Assert.assertEquals("MEMCACHED", c1.getName());
Assert.assertEquals("Servers-host_port", c1.getCompExports());
Assert.assertEquals(1, c1.getCommands().size());
ComponentCommand cmd = c1.getCommands().get(0);
Assert.assertEquals("START", cmd.getName());
Assert.assertEquals("SHELL", cmd.getType());
Assert.assertEquals("java -classpath /usr/myapps/memcached/*:/usr/lib/hadoop/lib/* com.thimbleware.jmemcached.Main",
cmd.getExec());
Component c2 = mInfo.getApplicationComponent("MEMCACHED2");
Assert.assertNotNull(c2);
Assert.assertEquals("MEMCACHED2", c2.getName());
Assert.assertEquals(1, c2.getCommands().size());
cmd = c2.getCommands().get(0);
Assert.assertEquals("CONFIGURE", cmd.getName());
Assert.assertEquals("PYTHON", cmd.getType());
Assert.assertEquals("scripts/config.py", cmd.getExec());
}
}

60
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
View File

@ -1,60 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.publisher;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.providers.agent.AgentProviderService;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
*
*/
public class TestAgentProviderService extends AgentProviderService {
protected static final Logger log =
LoggerFactory.getLogger(TestAgentProviderService.class);
public TestAgentProviderService() {
super();
log.info("TestAgentProviderService created");
}
@Override
public void bind(StateAccessForProviders stateAccessor,
QueueAccess queueAccess,
List<Container> liveContainers) {
super.bind(stateAccessor, queueAccess, liveContainers);
Map<String,String> dummyProps = new HashMap<String, String>();
dummyProps.put("prop1", "val1");
dummyProps.put("prop2", "val2");
log.info("publishing dummy-site.xml with values {}", dummyProps);
publishApplicationInstanceData("dummy-site", "dummy configuration",
dummyProps.entrySet());
// publishing global config for testing purposes
publishApplicationInstanceData("global", "global configuration",
stateAccessor.getAppConfSnapshot()
.getGlobalOptions().entrySet());
}
}

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java
View File

@ -1,40 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.appmaster.web.rest.publisher;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.agent.AgentProviderFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
*/
public class TestSliderProviderFactory extends AgentProviderFactory{
protected static final Logger log =
LoggerFactory.getLogger(TestSliderProviderFactory.class);
public TestSliderProviderFactory() {
log.info("Created TestSliderProviderFactory");
}
@Override
public ProviderService createServerProvider() {
log.info("Creating TestAgentProviderService");
return new TestAgentProviderService();
}
}