From 234dba84a1dcdadc0fd84f585ce4c61cf9f257fe Mon Sep 17 00:00:00 2001 From: Jian He Date: Tue, 29 Nov 2016 13:06:45 -0800 Subject: [PATCH] YARN-5909. Remove agent related code in slider AM. Contributed by Jian He --- hadoop-project/pom.xml | 5 - .../services/webapp/ApplicationApiWebApp.java | 2 +- .../hadoop-yarn-slider-core/pom.xml | 23 +- .../slider/common/tools/SliderUtils.java | 1 - .../providers/AbstractProviderService.java | 21 +- .../slider/providers/ProviderService.java | 9 +- .../providers/agent/AgentClientProvider.java | 713 ----- .../providers/agent/AgentLaunchParameter.java | 130 - .../providers/agent/AgentProviderFactory.java | 47 - .../providers/agent/AgentProviderService.java | 2850 ----------------- .../slider/providers/agent/AgentRoles.java | 38 - .../slider/providers/agent/AgentUtils.java | 150 - .../slider/providers/agent/Command.java | 59 - .../slider/providers/agent/CommandResult.java | 40 - .../agent/ComponentCommandOrder.java | 225 -- .../agent/ComponentInstanceState.java | 340 -- .../providers/agent/ComponentTagProvider.java | 127 - .../providers/agent/ContainerState.java | 41 - .../providers/agent/HeartbeatMonitor.java | 130 - .../apache/slider/providers/agent/State.java | 199 -- .../metadata/AbstractComponent.java | 80 - .../metadata/AbstractMetainfoParser.java | 130 - .../metadata/AbstractMetainfoSchema.java | 69 - .../metadata/AddonPackageMetainfoParser.java | 53 - .../application/metadata/Application.java | 193 -- .../metadata/ApplicationPackage.java | 69 - .../application/metadata/CommandOrder.java | 61 - .../application/metadata/CommandScript.java | 72 - .../agent/application/metadata/Component.java | 217 -- .../metadata/ComponentCommand.java | 85 - .../application/metadata/ComponentExport.java | 54 - .../metadata/ComponentsInAddonPackage.java | 26 - .../application/metadata/ConfigFile.java | 59 - .../application/metadata/DefaultConfig.java | 39 - .../metadata/DefaultConfigParser.java | 54 - .../application/metadata/DockerContainer.java | 187 -- .../metadata/DockerContainerInputFile.java | 50 - .../metadata/DockerContainerMount.java | 60 - .../metadata/DockerContainerPort.java | 66 - .../agent/application/metadata/Export.java | 61 - .../application/metadata/ExportGroup.java | 71 - .../agent/application/metadata/Metainfo.java | 118 - .../application/metadata/MetainfoParser.java | 97 - .../agent/application/metadata/OSPackage.java | 51 - .../application/metadata/OSSpecific.java | 57 - .../agent/application/metadata/Package.java | 60 - .../application/metadata/PropertyInfo.java | 54 - .../agent/application/metadata/Validate.java | 27 - .../org/apache/slider/providers/agent/todo.md | 22 - .../server/appmaster/SliderAppMaster.java | 140 +- .../server/appmaster/web/AgentService.java | 37 - .../server/appmaster/web/WebAppApi.java | 7 - .../server/appmaster/web/WebAppApiImpl.java | 6 - .../appmaster/web/rest/AMWebServices.java | 1 - .../web/rest/agent/AgentCommandType.java | 23 - .../appmaster/web/rest/agent/AgentEnv.java | 376 --- .../web/rest/agent/AgentResource.java | 118 - .../web/rest/agent/AgentRestOperations.java | 28 - .../appmaster/web/rest/agent/AgentWebApp.java | 258 -- .../web/rest/agent/AgentWebServices.java | 40 - .../web/rest/agent/CommandReport.java | 207 -- .../web/rest/agent/ComponentStatus.java | 129 - .../appmaster/web/rest/agent/DiskInfo.java | 128 - .../web/rest/agent/ExecutionCommand.java | 310 -- .../appmaster/web/rest/agent/HeartBeat.java | 149 - .../web/rest/agent/HeartBeatResponse.java | 147 - .../appmaster/web/rest/agent/HostInfo.java | 398 --- .../appmaster/web/rest/agent/HostStatus.java | 63 - .../appmaster/web/rest/agent/Register.java | 193 -- .../web/rest/agent/RegistrationCommand.java | 43 - .../web/rest/agent/RegistrationResponse.java | 133 - .../web/rest/agent/RegistrationStatus.java | 22 - .../web/rest/agent/StatusCommand.java | 152 - .../agent/TestAgentClientProvider.java | 77 - .../agent/TestAgentLaunchParameter.java | 76 - .../providers/agent/TestAgentUtils.java | 94 - .../agent/TestAppDefinitionPersister.java | 264 -- .../agent/TestComponentTagProvider.java | 115 - .../slider/providers/agent/TestState.java | 33 - .../metadata/TestConfigParser.java | 107 - .../metadata/TestMetainfoParser.java | 177 - .../publisher/TestAgentProviderService.java | 60 - .../publisher/TestSliderProviderFactory.java | 40 - 83 files changed, 26 insertions(+), 11517 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderFactory.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/Command.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentTagProvider.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/State.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentRestOperations.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebServices.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/DiskInfo.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostInfo.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostStatus.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationCommand.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index d803d13c937..b65f00a7132 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -965,11 +965,6 @@ jackson-module-jaxb-annotations ${jackson2.version} - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - ${jackson2.version} - com.fasterxml.jackson.dataformat jackson-dataformat-cbor diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java index 52a9de6b46d..e1bddb5fcfb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java @@ -30,7 +30,7 @@ import org.apache.hadoop.yarn.services.api.impl.ApplicationApiService; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; -import org.mortbay.jetty.webapp.Configuration; +import org.eclipse.jetty.webapp.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml index a2c67c0add1..66e9ee9283b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml @@ -225,7 +225,7 @@ javax.servlet - servlet-api + javax.servlet-api @@ -288,21 +288,6 @@ test - - org.mortbay.jetty - jetty - - - - org.mortbay.jetty - jetty-util - - - - org.mortbay.jetty - jetty-sslengine - - javax.servlet.jsp jsp-api @@ -314,12 +299,6 @@ jettison - - org.mortbay.jetty - jetty-sslengine - 6.1.26 - - org.yaml snakeyaml diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 713cd02d86c..b101d34349d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -70,7 +70,6 @@ import org.apache.slider.core.launch.ClasspathConstructor; import org.apache.slider.core.main.LauncherExitCodes; import org.apache.slider.providers.agent.AgentKeys; -import org.apache.slider.providers.agent.application.metadata.Component; import org.apache.slider.server.services.utility.PatternValidator; import org.apache.slider.server.services.workflow.ForkedProcessService; import org.apache.zookeeper.server.util.KerberosUtil; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java index 19fa07bcee5..00fc606d9ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java @@ -19,17 +19,17 @@ package org.apache.slider.providers; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; +import org.apache.hadoop.registry.client.exceptions.InvalidRecordException; +import org.apache.hadoop.registry.client.types.AddressTypes; +import org.apache.hadoop.registry.client.types.Endpoint; +import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.service.Service; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; -import org.apache.hadoop.registry.client.exceptions.InvalidRecordException; -import org.apache.hadoop.registry.client.types.AddressTypes; -import org.apache.hadoop.registry.client.types.Endpoint; -import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.slider.api.ClusterDescription; import org.apache.slider.common.SliderKeys; import org.apache.slider.common.tools.ConfigHelper; @@ -44,7 +44,6 @@ import org.apache.slider.server.appmaster.state.ContainerReleaseSelector; import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector; import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations; import org.apache.slider.server.services.workflow.ForkedProcessService; import org.apache.slider.server.services.workflow.ServiceParent; import org.apache.slider.server.services.workflow.WorkflowSequenceService; @@ -77,7 +76,6 @@ public abstract class AbstractProviderService private static final Logger log = LoggerFactory.getLogger(AbstractProviderService.class); protected StateAccessForProviders amState; - protected AgentRestOperations restOps; protected URL amWebAPI; protected YarnRegistryViewForProviders yarnRegistry; protected QueueAccess queueAccess; @@ -126,19 +124,10 @@ public YarnRegistryViewForProviders getYarnRegistry() { return yarnRegistry; } - @Override - public AgentRestOperations getAgentRestOperations() { - return restOps; - } - @Override public void notifyContainerCompleted(ContainerId containerId) { } - public void setAgentRestOperations(AgentRestOperations agentRestOperations) { - this.restOps = agentRestOperations; - } - /** * Load default Configuration * @param confDir configuration directory diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java index b62510a9579..4ca9326e911 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java @@ -20,10 +20,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.service.Service; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.slider.api.ClusterDescription; import org.apache.slider.common.tools.SliderFileSystem; @@ -37,7 +37,6 @@ import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions; import org.apache.slider.server.appmaster.state.ContainerReleaseSelector; import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations; import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; import java.io.File; @@ -175,12 +174,6 @@ public void bind(StateAccessForProviders stateAccessor, */ void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry); - /** - * Returns the agent rest operations interface. - * @return the interface if available, null otherwise. - */ - AgentRestOperations getAgentRestOperations(); - /** * Build up the endpoint details for this service * @param details diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java deleted file mode 100644 index 7ca469fbff4..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java +++ /dev/null @@ -1,713 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent; - -import com.google.common.io.Files; -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.hadoop.registry.client.binding.RegistryUtils; -import org.apache.slider.api.InternalKeys; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.client.ClientUtils; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.conf.MapOperations; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.launch.AbstractLauncher; -import org.apache.slider.core.registry.docstore.PublishedConfiguration; -import org.apache.slider.providers.AbstractClientProvider; -import org.apache.slider.providers.ProviderRole; -import org.apache.slider.providers.ProviderUtils; -import org.apache.slider.providers.agent.application.metadata.Application; -import org.apache.slider.providers.agent.application.metadata.Component; -import org.apache.slider.providers.agent.application.metadata.ConfigFile; -import org.apache.slider.providers.agent.application.metadata.Metainfo; -import org.apache.slider.providers.agent.application.metadata.MetainfoParser; -import org.apache.slider.providers.agent.application.metadata.OSPackage; -import org.apache.slider.providers.agent.application.metadata.OSSpecific; -import org.apache.slider.providers.agent.application.metadata.Package; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.Charset; -import java.util.Arrays; -import java.util.HashMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -import static org.apache.slider.common.tools.SliderUtils.getApplicationDefinitionPath; - -/** This class implements the client-side aspects of the agent deployer */ -public class AgentClientProvider extends AbstractClientProvider - implements AgentKeys, SliderKeys { - - - protected static final Logger log = - LoggerFactory.getLogger(AgentClientProvider.class); - protected static final String NAME = "agent"; - private static final ProviderUtils providerUtils = new ProviderUtils(log); - public static final String E_COULD_NOT_READ_METAINFO - = "Not a valid app package. Could not read metainfo."; - - protected Map metaInfoMap = new ConcurrentHashMap(); - - protected AgentClientProvider(Configuration conf) { - super(conf); - } - - @Override - public String getName() { - return NAME; - } - - @Override - public List getRoles() { - return AgentRoles.getRoles(); - } - - @Override //Client - public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem, - String clustername, - Configuration configuration, - AggregateConf instanceDefinition, - Path clusterDirPath, - Path generatedConfDirPath, - boolean secure) throws - SliderException, - IOException { - super.preflightValidateClusterConfiguration(sliderFileSystem, clustername, - configuration, - instanceDefinition, - clusterDirPath, - generatedConfDirPath, secure); - - String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition - .getAppConfOperations()); - Path appDefPath = new Path(appDef); - sliderFileSystem.verifyFileExists(appDefPath); - - String agentConf = instanceDefinition.getAppConfOperations(). - getGlobalOptions().getOption(AGENT_CONF, ""); - if (StringUtils.isNotEmpty(agentConf)) { - sliderFileSystem.verifyFileExists(new Path(agentConf)); - } - - String appHome = instanceDefinition.getAppConfOperations(). - getGlobalOptions().get(PACKAGE_PATH); - if (SliderUtils.isUnset(appHome)) { - String agentImage = instanceDefinition.getInternalOperations(). - get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); - sliderFileSystem.verifyFileExists(new Path(agentImage)); - } - } - - @Override - public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws - SliderException { - super.validateInstanceDefinition(instanceDefinition, fs); - log.debug("Validating conf {}", instanceDefinition); - ConfTreeOperations resources = - instanceDefinition.getResourceOperations(); - - providerUtils.validateNodeCount(instanceDefinition, ROLE_NODE, - 0, -1); - - String appDef = null; - try { - // Validate the app definition - appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition - .getAppConfOperations()); - } catch (BadConfigException bce) { - throw new BadConfigException("Application definition must be provided. " + bce.getMessage()); - } - - log.info("Validating app definition {}", appDef); - String extension = appDef.substring(appDef.lastIndexOf(".") + 1, appDef.length()); - if (!"zip".equals(extension.toLowerCase(Locale.ENGLISH))) { - throw new BadConfigException("App definition must be packaged as a .zip file. File provided is " + appDef); - } - - Set names = resources.getComponentNames(); - names.remove(COMPONENT_AM); - Map priorityMap = new HashMap(); - - for (String name : names) { - try { - // Validate the app definition - appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition - .getAppConfOperations(), name); - } catch (BadConfigException bce) { - throw new BadConfigException("Application definition must be provided. " + bce.getMessage()); - } - Metainfo metaInfo = getMetainfo(fs, appDef); - - MapOperations component = resources.getMandatoryComponent(name); - - if (metaInfo != null) { - Component componentDef = metaInfo.getApplicationComponent( - AgentUtils.getMetainfoComponentName(name, - instanceDefinition.getAppConfOperations())); - if (componentDef == null) { - throw new BadConfigException( - "Component %s is not a member of application.", name); - } - } - - int priority = - component.getMandatoryOptionInt(ResourceKeys.COMPONENT_PRIORITY); - if (priority <= 0) { - throw new BadConfigException("Component %s %s value out of range %d", - name, - ResourceKeys.COMPONENT_PRIORITY, - priority); - } - - String existing = priorityMap.get(priority); - if (existing != null) { - throw new BadConfigException( - "Component %s has a %s value %d which duplicates that of %s", - name, - ResourceKeys.COMPONENT_PRIORITY, - priority, - existing); - } - priorityMap.put(priority, name); - - // fileSystem may be null for tests - if (metaInfo != null) { - Component componentDef = metaInfo.getApplicationComponent( - AgentUtils.getMetainfoComponentName(name, - instanceDefinition.getAppConfOperations())); - - // ensure that intance count is 0 for client components - if ("CLIENT".equals(componentDef.getCategory())) { - MapOperations componentConfig = resources.getMandatoryComponent(name); - int count = - componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES); - if (count > 0) { - throw new BadConfigException("Component %s is of type CLIENT and cannot be instantiated." - + " Use \"slider client install ...\" command instead.", - name); - } - } else { - MapOperations componentConfig = resources.getMandatoryComponent(name); - int count = - componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES); - int definedMinCount = componentDef.getMinInstanceCountInt(); - int definedMaxCount = componentDef.getMaxInstanceCountInt(); - if (count < definedMinCount || count > definedMaxCount) { - throw new BadConfigException("Component %s, %s value %d out of range. " - + "Expected minimum is %d and maximum is %d", - name, - ResourceKeys.COMPONENT_INSTANCES, - count, - definedMinCount, - definedMaxCount); - } - } - } - } - } - - - @Override - public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem, - Configuration serviceConf, - AbstractLauncher launcher, - AggregateConf instanceDefinition, - Path snapshotConfDirPath, - Path generatedConfDirPath, - Configuration clientConfExtras, - String libdir, - Path tempPath, - boolean miniClusterTestRun) throws - IOException, - SliderException { - String agentImage = instanceDefinition.getInternalOperations(). - get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); - if (SliderUtils.isUnset(agentImage)) { - Path agentPath = new Path(tempPath.getParent(), PROVIDER_AGENT); - log.info("Automatically uploading the agent tarball at {}", agentPath); - fileSystem.getFileSystem().mkdirs(agentPath); - if (ProviderUtils.addAgentTar(this, AGENT_TAR, fileSystem, agentPath)) { - instanceDefinition.getInternalOperations().set( - InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, - new Path(agentPath, AGENT_TAR).toUri()); - } - } - } - - @Override - public Set getApplicationTags(SliderFileSystem fileSystem, - ConfTreeOperations appConf) throws SliderException { - return getApplicationTags(fileSystem, - getApplicationDefinitionPath(appConf)); - } - - public Set getApplicationTags(SliderFileSystem fileSystem, - String appDef) throws SliderException { - Set tags; - Metainfo metaInfo = getMetainfo(fileSystem, appDef); - - if (metaInfo == null) { - log.error("Error retrieving metainfo from {}", appDef); - throw new SliderException("Error parsing metainfo file, possibly bad structure."); - } - - Application application = metaInfo.getApplication(); - tags = new HashSet(); - tags.add("Name: " + application.getName()); - tags.add("Version: " + application.getVersion()); - tags.add("Description: " + SliderUtils.truncate(application.getComment(), 80)); - - return tags; - } - - @Override - public void processClientOperation(SliderFileSystem fileSystem, - RegistryOperations rops, - Configuration configuration, - String operation, - File clientInstallPath, - File appPackage, - JSONObject config, - String name) throws SliderException { - // create temp folder - // create sub-folders app_pkg, agent_pkg, command - File tmpDir = Files.createTempDir(); - log.info("Command is being executed at {}", tmpDir.getAbsolutePath()); - File appPkgDir = new File(tmpDir, "app_pkg"); - appPkgDir.mkdir(); - - File agentPkgDir = new File(tmpDir, "agent_pkg"); - agentPkgDir.mkdir(); - - File cmdDir = new File(tmpDir, "command"); - cmdDir.mkdir(); - - Metainfo metaInfo = null; - JSONObject defaultConfig = null; - try { - // expand app package into /app_pkg - ZipInputStream zipInputStream = null; - try { - zipInputStream = new ZipInputStream(new FileInputStream(appPackage)); - { - ZipEntry zipEntry = zipInputStream.getNextEntry(); - while (zipEntry != null) { - log.info("Processing {}", zipEntry.getName()); - String filePath = appPkgDir + File.separator + zipEntry.getName(); - if (!zipEntry.isDirectory()) { - log.info("Extracting file {}", filePath); - extractFile(zipInputStream, filePath); - - if ("metainfo.xml".equals(zipEntry.getName())) { - FileInputStream input = null; - try { - input = new FileInputStream(filePath); - metaInfo = new MetainfoParser().fromXmlStream(input); - } finally { - IOUtils.closeStream(input); - } - } else if ("metainfo.json".equals(zipEntry.getName())) { - FileInputStream input = null; - try { - input = new FileInputStream(filePath); - metaInfo = new MetainfoParser().fromJsonStream(input); - } finally { - IOUtils.closeStream(input); - } - } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) { - try { - defaultConfig = new JSONObject(FileUtils.readFileToString(new File(filePath), Charset.defaultCharset())); - } catch (JSONException jex) { - throw new SliderException("Unable to read default client config.", jex); - } - } - } else { - log.info("Creating dir {}", filePath); - File dir = new File(filePath); - dir.mkdir(); - } - zipInputStream.closeEntry(); - zipEntry = zipInputStream.getNextEntry(); - } - } - } finally { - zipInputStream.close(); - } - - if (metaInfo == null) { - throw new BadConfigException(E_COULD_NOT_READ_METAINFO); - } - - String clientScript = null; - String clientComponent = null; - for (Component component : metaInfo.getApplication().getComponents()) { - if (component.getCategory().equals("CLIENT")) { - clientComponent = component.getName(); - if (component.getCommandScript() != null) { - clientScript = component.getCommandScript().getScript(); - } - break; - } - } - - if (SliderUtils.isUnset(clientScript)) { - log.info("Installing CLIENT without script"); - List packages = metaInfo.getApplication().getPackages(); - if (packages.size() > 0) { - // retrieve package resources from HDFS and extract - for (Package pkg : packages) { - Path pkgPath = fileSystem.buildResourcePath(pkg.getName()); - if (!fileSystem.isFile(pkgPath) && name != null) { - pkgPath = fileSystem.buildResourcePath(name, pkg.getName()); - } - if (!fileSystem.isFile(pkgPath)) { - throw new IOException("Package doesn't exist as a resource: " + - pkg.getName()); - } - if ("archive".equals(pkg.getType())) { - File pkgFile = new File(tmpDir, pkg.getName()); - fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile); - expandTar(pkgFile, clientInstallPath); - } else { - File pkgFile = new File(clientInstallPath, pkg.getName()); - fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile); - } - } - } else { - // extract tarball from app def - for (OSSpecific osSpecific : metaInfo.getApplication() - .getOSSpecifics()) { - for (OSPackage pkg : osSpecific.getPackages()) { - if ("tarball".equals(pkg.getType())) { - File pkgFile = new File(appPkgDir, pkg.getName()); - expandTar(pkgFile, clientInstallPath); - } - } - } - } - if (name == null) { - log.warn("Conf files not being generated because no app name was " + - "provided"); - return; - } - File confInstallDir; - String clientRoot = null; - if (config != null) { - try { - clientRoot = config.getJSONObject("global") - .getString(APP_CLIENT_ROOT); - } catch (JSONException e) { - log.info("Couldn't read {} from provided client config, falling " + - "back on default", APP_CLIENT_ROOT); - } - } - if (clientRoot == null && defaultConfig != null) { - try { - clientRoot = defaultConfig.getJSONObject("global") - .getString(APP_CLIENT_ROOT); - } catch (JSONException e) { - log.info("Couldn't read {} from default client config, using {}", - APP_CLIENT_ROOT, clientInstallPath); - } - } - if (clientRoot == null) { - confInstallDir = clientInstallPath; - } else { - confInstallDir = new File(new File(clientInstallPath, clientRoot), "conf"); - if (!confInstallDir.exists()) { - confInstallDir.mkdirs(); - } - } - String user = RegistryUtils.currentUser(); - for (ConfigFile configFile : metaInfo.getComponentConfigFiles(clientComponent)) { - retrieveConfigFile(rops, configuration, configFile, name, user, - confInstallDir); - } - } else { - log.info("Installing CLIENT using script {}", clientScript); - expandAgentTar(agentPkgDir); - - JSONObject commandJson = getCommandJson(defaultConfig, config, metaInfo, clientInstallPath, name); - FileWriter file = new FileWriter(new File(cmdDir, "command.json")); - try { - file.write(commandJson.toString()); - - } catch (IOException e) { - log.error("Couldn't write command.json to file"); - } finally { - file.flush(); - file.close(); - } - - runCommand(appPkgDir, agentPkgDir, cmdDir, clientScript); - } - - } catch (IOException ioex) { - log.warn("Error while executing INSTALL command {}", ioex.getMessage()); - throw new SliderException("INSTALL client failed."); - } - } - - protected void runCommand( - File appPkgDir, - File agentPkgDir, - File cmdDir, - String clientScript) throws SliderException { - int exitCode = 0; - Exception exp = null; - try { - String clientScriptPath = appPkgDir.getAbsolutePath() + File.separator + "package" + - File.separator + clientScript; - List command = Arrays.asList(PYTHON_EXE, - "-S", - clientScriptPath, - "INSTALL", - cmdDir.getAbsolutePath() + File.separator + "command.json", - appPkgDir.getAbsolutePath() + File.separator + "package", - cmdDir.getAbsolutePath() + File.separator + "command-out.json", - "DEBUG"); - ProcessBuilder pb = new ProcessBuilder(command); - log.info("Command: " + StringUtils.join(pb.command(), " ")); - pb.environment().put(PYTHONPATH, - agentPkgDir.getAbsolutePath() - + File.separator + "slider-agent" + File.pathSeparator - + agentPkgDir.getAbsolutePath() - + File.separator + "slider-agent/jinja2"); - log.info("{}={}", PYTHONPATH, pb.environment().get(PYTHONPATH)); - - Process proc = pb.start(); - InputStream stderr = proc.getErrorStream(); - InputStream stdout = proc.getInputStream(); - BufferedReader stdOutReader = new BufferedReader(new InputStreamReader(stdout)); - BufferedReader stdErrReader = new BufferedReader(new InputStreamReader(stderr)); - - proc.waitFor(); - - String line; - while ((line = stdOutReader.readLine()) != null) { - log.info("Stdout: " + line); - } - while ((line = stdErrReader.readLine()) != null) { - log.info("Stderr: " + line); - } - - exitCode = proc.exitValue(); - log.info("Exit value is {}", exitCode); - } catch (IOException e) { - exp = e; - } catch (InterruptedException e) { - exp = e; - } - - if (exitCode != 0) { - throw new SliderException("INSTALL client failed with exit code " + exitCode); - } - - if (exp != null) { - log.error("Error while executing INSTALL command {}. Stack trace {}", - exp.getMessage(), - ExceptionUtils.getStackTrace(exp)); - throw new SliderException("INSTALL client failed.", exp); - } - } - - private void expandAgentTar(File agentPkgDir) throws IOException { - String libDirProp = SliderUtils.getLibDir(); - File tarFile = new File(libDirProp, AGENT_TAR); - expandTar(tarFile, agentPkgDir); - } - - private void expandTar(File tarFile, File destDir) throws IOException { - log.info("Expanding tar {} to {}", tarFile, destDir); - TarArchiveInputStream tarIn = new TarArchiveInputStream( - new GzipCompressorInputStream( - new BufferedInputStream( - new FileInputStream(tarFile) - ) - ) - ); - try { - TarArchiveEntry tarEntry = tarIn.getNextTarEntry(); - while (tarEntry != null) { - File destPath = new File(destDir, tarEntry.getName()); - File parent = destPath.getParentFile(); - if (!parent.exists()) { - parent.mkdirs(); - } - if (tarEntry.isDirectory()) { - destPath.mkdirs(); - } else { - byte[] byteToRead = new byte[1024]; - BufferedOutputStream buffOut = - new BufferedOutputStream(new FileOutputStream(destPath)); - try { - int len; - while ((len = tarIn.read(byteToRead)) != -1) { - buffOut.write(byteToRead, 0, len); - } - } finally { - buffOut.close(); - } - } - if ((tarEntry.getMode() & 0100) != 0) { - destPath.setExecutable(true); - } - tarEntry = tarIn.getNextTarEntry(); - } - } finally { - tarIn.close(); - } - } - - private void retrieveConfigFile(RegistryOperations rops, - Configuration configuration, ConfigFile configFile, String name, - String user, File destDir) throws IOException, SliderException { - log.info("Retrieving config {} to {}", configFile.getDictionaryName(), - destDir); - PublishedConfiguration published = ClientUtils.getConfigFromRegistry(rops, - configuration, configFile.getDictionaryName(), name, user, true); - ClientUtils.saveOrReturnConfig(published, configFile.getType(), - destDir, configFile.getFileName()); - } - - protected JSONObject getCommandJson(JSONObject defaultConfig, - JSONObject inputConfig, - Metainfo metainfo, - File clientInstallPath, - String name) throws SliderException { - try { - JSONObject pkgList = new JSONObject(); - pkgList.put(PACKAGE_LIST, - AgentProviderService.getPackageListFromApplication(metainfo.getApplication())); - JSONObject obj = new JSONObject(); - obj.put("hostLevelParams", pkgList); - - String user = RegistryUtils.currentUser(); - JSONObject configuration = new JSONObject(); - JSONObject global = new JSONObject(); - global.put("app_install_dir", clientInstallPath.getAbsolutePath()); - global.put("app_user", user); - if (name != null) { - global.put("app_name", name); - } - - if (defaultConfig != null) { - readConfigEntries(defaultConfig, clientInstallPath, global, name, user); - } - if (inputConfig != null) { - readConfigEntries(inputConfig, clientInstallPath, global, name, user); - } - - configuration.put("global", global); - obj.put("configurations", configuration); - return obj; - } catch (JSONException jex) { - log.warn("Error while executing INSTALL command {}", jex.getMessage()); - throw new SliderException("INSTALL client failed."); - } - } - - private void readConfigEntries(JSONObject inpConfig, - File clientInstallPath, - JSONObject globalConfig, - String name, String user) - throws JSONException { - JSONObject globalSection = inpConfig.getJSONObject("global"); - Iterator it = globalSection.keys(); - while (it.hasNext()) { - String key = (String) it.next(); - String value = globalSection.getString(key); - if (SliderUtils.isSet(value)) { - value = value.replace("{app_install_dir}", clientInstallPath.getAbsolutePath()); - value = value.replace("{app_user}", user); - if (name != null) { - value = value.replace("{app_name}", name); - } - } - if (globalConfig.has(key)) { - // last one wins - globalConfig.remove(key); - } - globalConfig.put(key, value); - } - } - - private void extractFile(ZipInputStream zipInputStream, String filePath) throws IOException { - BufferedOutputStream output = new BufferedOutputStream(new FileOutputStream(filePath)); - try { - byte[] bytesRead = new byte[4096]; - int read = 0; - while ((read = zipInputStream.read(bytesRead)) != -1) { - output.write(bytesRead, 0, read); - } - } finally { - output.close(); - } - } - - private Metainfo getMetainfo(SliderFileSystem fs, String appDef) { - Metainfo metaInfo = metaInfoMap.get(appDef); - if (fs != null && metaInfo == null) { - try { - metaInfo = AgentUtils.getApplicationMetainfo(fs, appDef, false); - metaInfoMap.put(appDef, metaInfo); - } catch (IOException ioe) { - // Ignore missing metainfo file for now - log.info("Missing metainfo {}", ioe.getMessage()); - } catch (BadConfigException bce) { - log.info("Bad Configuration {}", bce.getMessage()); - } - } - return metaInfo; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java deleted file mode 100644 index 18c6374e9fd..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Pattern; - -class AgentLaunchParameter { - public static final Logger log = - LoggerFactory.getLogger(AgentLaunchParameter.class); - private static final String DEFAULT_PARAMETER = ""; - private static final String ANY_COMPONENT = "ANY"; - private static final String NONE_VALUE = "NONE"; - private final Map launchParameterTracker; - - public AgentLaunchParameter(String parameters) { - launchParameterTracker = parseExpectedLaunchParameters(parameters); - } - - /** - * Get command for the component type - * - * @param componentGroup - * - * @return - */ - public String getNextLaunchParameter(String componentGroup) { - if (launchParameterTracker != null) { - if (launchParameterTracker.containsKey(componentGroup) - || launchParameterTracker.containsKey(ANY_COMPONENT)) { - synchronized (this) { - CommandTracker indexTracker = null; - if (launchParameterTracker.containsKey(componentGroup)) { - indexTracker = launchParameterTracker.get(componentGroup); - } else { - indexTracker = launchParameterTracker.get(ANY_COMPONENT); - } - - return indexTracker.getNextCommand(); - } - } - } - - return DEFAULT_PARAMETER; - } - - /** - * Parse launch parameters of the form ANY:PARAM_FOR_FIRST:PARAM_FOR_SECOND:...:PARAM_FOR_REST|HBASE_MASTER:... - * - * E.g. ANY:DO_NOT_REGISTER:DO_NOT_HEARTBEAT:NONE For any container, first one gets DO_NOT_REGISTER second one gets - * DO_NOT_HEARTBEAT, then all of the rest get nothing - * - * E.g. HBASE_MASTER:FAIL_AFTER_START:NONE For HBASE_MASTER, first one gets FAIL_AFTER_START then "" for all - * - * @param launchParameters - * - * @return - */ - Map parseExpectedLaunchParameters(String launchParameters) { - Map trackers = null; - if (launchParameters != null && launchParameters.length() > 0) { - String[] componentSpecificParameters = launchParameters.split(Pattern.quote("|")); - for (String componentSpecificParameter : componentSpecificParameters) { - if (componentSpecificParameter.length() != 0) { - String[] parameters = componentSpecificParameter.split(Pattern.quote(":")); - - if (parameters.length > 1 && parameters[0].length() > 0) { - - for (int index = 1; index < parameters.length; index++) { - if (parameters[index].equals(NONE_VALUE)) { - parameters[index] = DEFAULT_PARAMETER; - } - } - - if (trackers == null) { - trackers = new HashMap(10); - } - String componentName = parameters[0]; - CommandTracker tracker = new CommandTracker(Arrays.copyOfRange(parameters, 1, parameters.length)); - trackers.put(componentName, tracker); - } - } - } - } - - return trackers; - } - - class CommandTracker { - private final int maxIndex; - private final String[] launchCommands; - private int currentIndex; - - CommandTracker(String[] launchCommands) { - this.currentIndex = 0; - this.maxIndex = launchCommands.length - 1; - this.launchCommands = launchCommands; - } - - String getNextCommand() { - String retVal = launchCommands[currentIndex]; - if (currentIndex != maxIndex) { - currentIndex++; - } - - return retVal; - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderFactory.java deleted file mode 100644 index d5ca749a431..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderFactory.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent; - -import org.apache.hadoop.conf.Configuration; -import org.apache.slider.providers.AbstractClientProvider; -import org.apache.slider.providers.ProviderService; -import org.apache.slider.providers.SliderProviderFactory; - -public class AgentProviderFactory extends SliderProviderFactory { - - public static final String CLASSNAME = - "org.apache.slider.providers.agent.AgentProviderFactory"; - - public AgentProviderFactory() { - } - - public AgentProviderFactory(Configuration conf) { - super(conf); - } - - @Override - public AbstractClientProvider createClientProvider() { - return new AgentClientProvider(getConf()); - } - - @Override - public ProviderService createServerProvider() { - return new AgentProviderService(); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java deleted file mode 100644 index 499812e3e9b..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ /dev/null @@ -1,2850 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.registry.client.types.Endpoint; -import org.apache.hadoop.registry.client.types.ProtocolTypes; -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.slider.api.ClusterDescription; -import org.apache.slider.api.ClusterNode; -import org.apache.slider.api.InternalKeys; -import org.apache.slider.api.OptionKeys; -import org.apache.slider.api.ResourceKeys; -import org.apache.slider.api.StatusKeys; -import org.apache.slider.common.SliderExitCodes; -import org.apache.slider.common.SliderKeys; -import org.apache.slider.common.SliderXmlConfKeys; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.conf.MapOperations; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.core.exceptions.NoSuchNodeException; -import org.apache.slider.core.exceptions.SliderException; -import org.apache.slider.core.launch.CommandLineBuilder; -import org.apache.slider.core.launch.ContainerLauncher; -import org.apache.slider.core.registry.docstore.ConfigFormat; -import org.apache.slider.core.registry.docstore.ConfigUtils; -import org.apache.slider.core.registry.docstore.ExportEntry; -import org.apache.slider.core.registry.docstore.PublishedConfiguration; -import org.apache.slider.core.registry.docstore.PublishedExports; -import org.apache.slider.core.registry.info.CustomRegistryConstants; -import org.apache.slider.providers.AbstractProviderService; -import org.apache.slider.providers.MonitorDetail; -import org.apache.slider.providers.ProviderCore; -import org.apache.slider.providers.ProviderRole; -import org.apache.slider.providers.ProviderUtils; -import org.apache.slider.providers.agent.application.metadata.AbstractComponent; -import org.apache.slider.providers.agent.application.metadata.Application; -import org.apache.slider.providers.agent.application.metadata.CommandOrder; -import org.apache.slider.providers.agent.application.metadata.CommandScript; -import org.apache.slider.providers.agent.application.metadata.Component; -import org.apache.slider.providers.agent.application.metadata.ComponentCommand; -import org.apache.slider.providers.agent.application.metadata.ComponentExport; -import org.apache.slider.providers.agent.application.metadata.ComponentsInAddonPackage; -import org.apache.slider.providers.agent.application.metadata.ConfigFile; -import org.apache.slider.providers.agent.application.metadata.DefaultConfig; -import org.apache.slider.providers.agent.application.metadata.DockerContainer; -import org.apache.slider.providers.agent.application.metadata.Export; -import org.apache.slider.providers.agent.application.metadata.ExportGroup; -import org.apache.slider.providers.agent.application.metadata.Metainfo; -import org.apache.slider.providers.agent.application.metadata.OSPackage; -import org.apache.slider.providers.agent.application.metadata.OSSpecific; -import org.apache.slider.providers.agent.application.metadata.Package; -import org.apache.slider.providers.agent.application.metadata.PropertyInfo; -import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss; -import org.apache.slider.server.appmaster.actions.RegisterComponentInstance; -import org.apache.slider.server.appmaster.state.ContainerPriority; -import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType; -import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations; -import org.apache.slider.server.appmaster.web.rest.agent.CommandReport; -import org.apache.slider.server.appmaster.web.rest.agent.ComponentStatus; -import org.apache.slider.server.appmaster.web.rest.agent.ExecutionCommand; -import org.apache.slider.server.appmaster.web.rest.agent.HeartBeat; -import org.apache.slider.server.appmaster.web.rest.agent.HeartBeatResponse; -import org.apache.slider.server.appmaster.web.rest.agent.Register; -import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse; -import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus; -import org.apache.slider.server.appmaster.web.rest.agent.StatusCommand; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Scanner; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.regex.Pattern; - -import static org.apache.slider.api.RoleKeys.ROLE_PREFIX; -import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS; - -/** - * This class implements the server-side logic for application deployment through Slider application package - */ -public class AgentProviderService extends AbstractProviderService implements - ProviderCore, - AgentKeys, - SliderKeys, AgentRestOperations { - - - protected static final Logger log = - LoggerFactory.getLogger(AgentProviderService.class); - private static final ProviderUtils providerUtils = new ProviderUtils(log); - private static final String LABEL_MAKER = "___"; - private static final String CONTAINER_ID = "container_id"; - private static final String GLOBAL_CONFIG_TAG = "global"; - private static final String COMPONENT_TAG = "component"; - private static final String APPLICATION_TAG = "application"; - private static final String COMPONENT_DATA_TAG = "ComponentInstanceData"; - private static final String SHARED_PORT_TAG = "SHARED"; - private static final String PER_CONTAINER_TAG = "{PER_CONTAINER}"; - private static final int MAX_LOG_ENTRIES = 40; - private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000; - - private final Object syncLock = new Object(); - private final ComponentTagProvider tags = new ComponentTagProvider(); - private int heartbeatMonitorInterval = 0; - private AgentClientProvider clientProvider; - private AtomicInteger taskId = new AtomicInteger(0); - private volatile Map metaInfoMap = new HashMap<>(); - private SliderFileSystem fileSystem = null; - private Map defaultConfigs = null; - private ComponentCommandOrder commandOrder = new ComponentCommandOrder(); - private HeartbeatMonitor monitor; - private Boolean canAnyMasterPublish = null; - private AgentLaunchParameter agentLaunchParameter = null; - private String clusterName = null; - private boolean isInUpgradeMode; - private Set upgradeContainers = new HashSet(); - private boolean appStopInitiated; - - private final Map componentStatuses = - new ConcurrentHashMap(); - private final Map> componentInstanceData = - new ConcurrentHashMap>(); - private final Map>> exportGroups = - new ConcurrentHashMap>>(); - private final Map> allocatedPorts = - new ConcurrentHashMap>(); - private final Map packageMetainfo = - new ConcurrentHashMap(); - - private final Map logFolderExports = - Collections.synchronizedMap(new LinkedHashMap(MAX_LOG_ENTRIES, 0.75f, false) { - protected boolean removeEldestEntry(Map.Entry eldest) { - return size() > MAX_LOG_ENTRIES; - } - }); - private final Map workFolderExports = - Collections.synchronizedMap(new LinkedHashMap(MAX_LOG_ENTRIES, 0.75f, false) { - protected boolean removeEldestEntry(Map.Entry eldest) { - return size() > MAX_LOG_ENTRIES; - } - }); - private final Map> containerExportsMap = - new HashMap>(); - - private static class MetainfoHolder { - Metainfo metaInfo; - private Map defaultConfigs = null; - - public MetainfoHolder(Metainfo metaInfo, - Map defaultConfigs) { - this.metaInfo = metaInfo; - this.defaultConfigs = defaultConfigs; - } - } - - /** - * Create an instance of AgentProviderService - */ - public AgentProviderService() { - super("AgentProviderService"); - setAgentRestOperations(this); - setHeartbeatMonitorInterval(DEFAULT_HEARTBEAT_MONITOR_INTERVAL); - } - - @Override - public String getHumanName() { - return "Slider Agent"; - } - - @Override - public List getRoles() { - return AgentRoles.getRoles(); - } - - @Override - protected void serviceInit(Configuration conf) throws Exception { - super.serviceInit(conf); - clientProvider = new AgentClientProvider(conf); - } - - @Override - public void validateInstanceDefinition(AggregateConf instanceDefinition) - throws - SliderException { - clientProvider.validateInstanceDefinition(instanceDefinition, null); - - ConfTreeOperations resources = - instanceDefinition.getResourceOperations(); - - Set names = resources.getComponentNames(); - names.remove(COMPONENT_AM); - for (String name : names) { - Component componentDef = getApplicationComponent(name); - if (componentDef == null) { - // component member is validated elsewhere, so we don't need to throw - // an exception here - continue; - } - - MapOperations componentConfig = resources.getMandatoryComponent(name); - int count = - componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES); - int definedMinCount = componentDef.getMinInstanceCountInt(); - int definedMaxCount = componentDef.getMaxInstanceCountInt(); - if (count < definedMinCount || count > definedMaxCount) { - throw new BadConfigException("Component %s, %s value %d out of range. " - + "Expected minimum is %d and maximum is %d", - name, - ResourceKeys.COMPONENT_INSTANCES, - count, - definedMinCount, - definedMaxCount); - } - } - } - - // Reads the metainfo.xml in the application package and loads it - private void buildMetainfo(AggregateConf instanceDefinition, - SliderFileSystem fileSystem, - String roleGroup) - throws IOException, SliderException { - String mapKey = instanceDefinition.getAppConfOperations() - .getComponentOpt(roleGroup, ROLE_PREFIX, DEFAULT_METAINFO_MAP_KEY); - String appDef = SliderUtils.getApplicationDefinitionPath( - instanceDefinition.getAppConfOperations(), roleGroup); - MapOperations component = null; - if (roleGroup != null) { - component = instanceDefinition.getAppConfOperations().getComponent(roleGroup); - } - - MetainfoHolder metaInfoHolder = metaInfoMap.get(mapKey); - if (metaInfoHolder == null) { - synchronized (syncLock) { - if (this.fileSystem == null) { - this.fileSystem = fileSystem; - } - metaInfoHolder = metaInfoMap.get(mapKey); - if (metaInfoHolder == null) { - readAndSetHeartbeatMonitoringInterval(instanceDefinition); - initializeAgentDebugCommands(instanceDefinition); - - Metainfo metaInfo = getApplicationMetainfo(fileSystem, appDef, false); - log.info("Master package metainfo: {}", metaInfo.toString()); - if (metaInfo == null || metaInfo.getApplication() == null) { - log.error("metainfo.xml is unavailable or malformed at {}.", appDef); - throw new SliderException( - "metainfo.xml is required in app package."); - } - List commandOrders = metaInfo.getApplication() - .getCommandOrders(); - if (!DEFAULT_METAINFO_MAP_KEY.equals(mapKey)) { - for (Component comp : metaInfo.getApplication().getComponents()) { - comp.setName(mapKey + comp.getName()); - log.info("Modifying external metainfo component name to {}", - comp.getName()); - } - for (CommandOrder co : commandOrders) { - log.info("Adding prefix {} to command order {}", - mapKey, co); - co.setCommand(mapKey + co.getCommand()); - co.setRequires(mapKey + co.getRequires()); - } - } - log.debug("Merging command orders {} for {}", commandOrders, - roleGroup); - commandOrder.mergeCommandOrders(commandOrders, - instanceDefinition.getResourceOperations()); - Map defaultConfigs = - initializeDefaultConfigs(fileSystem, appDef, metaInfo); - metaInfoMap.put(mapKey, new MetainfoHolder(metaInfo, defaultConfigs)); - monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval()); - monitor.start(); - - // build a map from component to metainfo - String addonAppDefString = instanceDefinition.getAppConfOperations() - .getGlobalOptions().getOption(ADDONS, null); - if (component != null) { - addonAppDefString = component.getOption(ADDONS, addonAppDefString); - } - log.debug("All addon appdefs: {}", addonAppDefString); - if (addonAppDefString != null) { - Scanner scanner = new Scanner(addonAppDefString).useDelimiter(","); - while (scanner.hasNext()) { - String addonAppDef = scanner.next(); - String addonAppDefPath = instanceDefinition - .getAppConfOperations().getGlobalOptions().get(addonAppDef); - if (component != null) { - addonAppDefPath = component.getOption(addonAppDef, addonAppDefPath); - } - log.debug("Addon package {} is stored at: {}", addonAppDef - + addonAppDefPath); - Metainfo addonMetaInfo = getApplicationMetainfo(fileSystem, - addonAppDefPath, true); - addonMetaInfo.validate(); - packageMetainfo.put(addonMetaInfo.getApplicationPackage() - .getName(), addonMetaInfo); - } - log.info("Metainfo map for master and addon: {}", - packageMetainfo.toString()); - } - } - } - } - } - - @Override - public void initializeApplicationConfiguration( - AggregateConf instanceDefinition, SliderFileSystem fileSystem, - String roleGroup) - throws IOException, SliderException { - buildMetainfo(instanceDefinition, fileSystem, roleGroup); - } - - @Override - public void buildContainerLaunchContext(ContainerLauncher launcher, - AggregateConf instanceDefinition, - Container container, - ProviderRole providerRole, - SliderFileSystem fileSystem, - Path generatedConfPath, - MapOperations resourceComponent, - MapOperations appComponent, - Path containerTmpDirPath) throws - IOException, - SliderException { - - String roleName = providerRole.name; - String roleGroup = providerRole.group; - String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition - .getAppConfOperations(), roleGroup); - - initializeApplicationConfiguration(instanceDefinition, fileSystem, roleGroup); - - log.info("Build launch context for Agent"); - log.debug(instanceDefinition.toString()); - - //if we are launching docker based app on yarn, then we need to pass docker image - if (isYarnDockerContainer(roleGroup)) { - launcher.setYarnDockerMode(true); - launcher.setDockerImage(getConfigFromMetaInfo(roleGroup, "image")); - launcher.setDockerNetwork(getConfigFromMetaInfo(roleGroup, "network")); - launcher.setRunPrivilegedContainer(getConfigFromMetaInfo(roleGroup, "runPriviledgedContainer")); - launcher - .setYarnContainerMountPoints(getConfigFromMetaInfoWithAppConfigOverriding( - roleGroup, "yarn.container.mount.points")); - } - - // Set the environment - launcher.putEnv(SliderUtils.buildEnvMap(appComponent, - providerUtils.getStandardTokenMap(getAmState().getAppConfSnapshot(), - getAmState().getInternalsSnapshot(), roleName, roleGroup, - getClusterName()))); - - String workDir = ApplicationConstants.Environment.PWD.$(); - launcher.setEnv("AGENT_WORK_ROOT", workDir); - log.info("AGENT_WORK_ROOT set to {}", workDir); - String logDir = ApplicationConstants.LOG_DIR_EXPANSION_VAR; - launcher.setEnv("AGENT_LOG_ROOT", logDir); - log.info("AGENT_LOG_ROOT set to {}", logDir); - if (System.getenv(HADOOP_USER_NAME) != null) { - launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME)); - } - // for 2-Way SSL - launcher.setEnv(SLIDER_PASSPHRASE, instanceDefinition.getPassphrase()); - //add english env - launcher.setEnv("LANG", "en_US.UTF-8"); - launcher.setEnv("LC_ALL", "en_US.UTF-8"); - launcher.setEnv("LANGUAGE", "en_US.UTF-8"); - - //local resources - - // TODO: Should agent need to support App Home - String scriptPath = new File(AGENT_MAIN_SCRIPT_ROOT, AGENT_MAIN_SCRIPT).getPath(); - String appHome = instanceDefinition.getAppConfOperations(). - getGlobalOptions().get(PACKAGE_PATH); - if (SliderUtils.isSet(appHome)) { - scriptPath = new File(appHome, AGENT_MAIN_SCRIPT).getPath(); - } - - // set PYTHONPATH - List pythonPaths = new ArrayList(); - pythonPaths.add(AGENT_MAIN_SCRIPT_ROOT); - pythonPaths.add(AGENT_JINJA2_ROOT); - String pythonPath = StringUtils.join(File.pathSeparator, pythonPaths); - launcher.setEnv(PYTHONPATH, pythonPath); - log.info("PYTHONPATH set to {}", pythonPath); - - Path agentImagePath = null; - String agentImage = instanceDefinition.getInternalOperations(). - get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH); - if (SliderUtils.isUnset(agentImage)) { - agentImagePath = - new Path(new Path(new Path(instanceDefinition.getInternalOperations().get(InternalKeys.INTERNAL_TMP_DIR), - container.getId().getApplicationAttemptId().getApplicationId().toString()), - PROVIDER_AGENT), - AGENT_TAR); - } else { - agentImagePath = new Path(agentImage); - } - - if (fileSystem.getFileSystem().exists(agentImagePath)) { - LocalResource agentImageRes = fileSystem.createAmResource(agentImagePath, LocalResourceType.ARCHIVE); - launcher.addLocalResource(AGENT_INSTALL_DIR, agentImageRes); - } else { - String msg = - String.format("Required agent image slider-agent.tar.gz is unavailable at %s", agentImagePath.toString()); - MapOperations compOps = appComponent; - boolean relaxVerificationForTest = compOps != null ? Boolean.valueOf(compOps. - getOptionBool(TEST_RELAX_VERIFICATION, false)) : false; - log.error(msg); - - if (!relaxVerificationForTest) { - throw new SliderException(SliderExitCodes.EXIT_DEPLOYMENT_FAILED, msg); - } - } - - log.info("Using {} for agent.", scriptPath); - LocalResource appDefRes = fileSystem.createAmResource( - fileSystem.getFileSystem().resolvePath(new Path(appDef)), - LocalResourceType.ARCHIVE); - launcher.addLocalResource(APP_DEFINITION_DIR, appDefRes); - - for (Package pkg : getMetaInfo(roleGroup).getApplication().getPackages()) { - Path pkgPath = fileSystem.buildResourcePath(pkg.getName()); - if (!fileSystem.isFile(pkgPath)) { - pkgPath = fileSystem.buildResourcePath(getClusterName(), - pkg.getName()); - } - if (!fileSystem.isFile(pkgPath)) { - throw new IOException("Package doesn't exist as a resource: " + - pkg.getName()); - } - log.info("Adding resource {}", pkg.getName()); - LocalResourceType type = LocalResourceType.FILE; - if ("archive".equals(pkg.getType())) { - type = LocalResourceType.ARCHIVE; - } - LocalResource packageResource = fileSystem.createAmResource( - pkgPath, type); - launcher.addLocalResource(APP_PACKAGES_DIR, packageResource); - } - - String agentConf = instanceDefinition.getAppConfOperations(). - getGlobalOptions().getOption(AGENT_CONF, ""); - if (SliderUtils.isSet(agentConf)) { - LocalResource agentConfRes = fileSystem.createAmResource(fileSystem - .getFileSystem().resolvePath(new Path(agentConf)), - LocalResourceType.FILE); - launcher.addLocalResource(AGENT_CONFIG_FILE, agentConfRes); - } - - String agentVer = instanceDefinition.getAppConfOperations(). - getGlobalOptions().getOption(AGENT_VERSION, null); - if (agentVer != null) { - LocalResource agentVerRes = fileSystem.createAmResource( - fileSystem.getFileSystem().resolvePath(new Path(agentVer)), - LocalResourceType.FILE); - launcher.addLocalResource(AGENT_VERSION_FILE, agentVerRes); - } - - if (SliderUtils.isHadoopClusterSecure(getConfig())) { - providerUtils.localizeServiceKeytabs(launcher, instanceDefinition, - fileSystem, getClusterName()); - } - - MapOperations amComponent = instanceDefinition. - getAppConfOperations().getComponent(COMPONENT_AM); - if (providerUtils.hasTwoWaySSLEnabled(amComponent)) { - providerUtils.localizeContainerSSLResources(launcher, container, - fileSystem, getClusterName()); - } - - if (providerUtils.areStoresRequested(appComponent)) { - providerUtils.localizeContainerSecurityStores(launcher, container, - roleName, fileSystem, instanceDefinition, appComponent, - getClusterName()); - } - - //add the configuration resources - launcher.addLocalResources(fileSystem.submitDirectory( - generatedConfPath, - PROPAGATED_CONF_DIR_NAME)); - - if (appComponent.getOptionBool(AM_CONFIG_GENERATION, false)) { - // build and localize configuration files - Map> configurations = - buildCommandConfigurations(instanceDefinition.getAppConfOperations(), - instanceDefinition.getInternalOperations(), - container.getId().toString(), roleName, roleGroup); - for (ConfigFile configFile : getMetaInfo(roleGroup) - .getComponentConfigFiles(roleGroup)) { - localizeConfigFile(launcher, roleName, roleGroup, configFile, - configurations, launcher.getEnv(), fileSystem); - } - } - - String label = getContainerLabel(container, roleName, roleGroup); - CommandLineBuilder operation = new CommandLineBuilder(); - - String pythonExec = instanceDefinition.getAppConfOperations() - .getGlobalOptions().getOption(SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH, - PYTHON_EXE); - - operation.add(pythonExec); - - operation.add(scriptPath); - operation.add(ARG_LABEL, label); - operation.add(ARG_ZOOKEEPER_QUORUM); - operation.add(getClusterOptionPropertyValue(OptionKeys.ZOOKEEPER_QUORUM)); - operation.add(ARG_ZOOKEEPER_REGISTRY_PATH); - operation.add(getZkRegistryPath()); - - String debugCmd = agentLaunchParameter.getNextLaunchParameter(roleGroup); - if (SliderUtils.isSet(debugCmd)) { - operation.add(ARG_DEBUG); - operation.add(debugCmd); - } - - operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" - + AGENT_OUT_FILE + " 2>&1"); - - launcher.addCommand(operation.build()); - - // localize addon package - String addonAppDefString = instanceDefinition.getAppConfOperations() - .getGlobalOptions().getOption(ADDONS, null); - log.debug("All addon appdefs: {}", addonAppDefString); - if (addonAppDefString != null) { - Scanner scanner = new Scanner(addonAppDefString).useDelimiter(","); - while (scanner.hasNext()) { - String addonAppDef = scanner.next(); - String addonAppDefPath = instanceDefinition - .getAppConfOperations().getGlobalOptions().get(addonAppDef); - log.debug("Addon package {} is stored at: {}", addonAppDef, addonAppDefPath); - LocalResource addonPkgRes = fileSystem.createAmResource( - fileSystem.getFileSystem().resolvePath(new Path(addonAppDefPath)), - LocalResourceType.ARCHIVE); - launcher.addLocalResource(ADDON_DEFINITION_DIR + "/" + addonAppDef, addonPkgRes); - } - log.debug("Metainfo map for master and addon: {}", - packageMetainfo.toString()); - } - - // Additional files to localize in addition to the application def - String appResourcesString = instanceDefinition.getAppConfOperations() - .getGlobalOptions().getOption(APP_RESOURCES, null); - log.info("Configuration value for extra resources to localize: {}", appResourcesString); - if (null != appResourcesString) { - try (Scanner scanner = new Scanner(appResourcesString).useDelimiter(",")) { - while (scanner.hasNext()) { - String resource = scanner.next(); - Path resourcePath = new Path(resource); - LocalResource extraResource = fileSystem.createAmResource( - fileSystem.getFileSystem().resolvePath(resourcePath), - LocalResourceType.FILE); - String destination = APP_RESOURCES_DIR + "/" + resourcePath.getName(); - log.info("Localizing {} to {}", resourcePath, destination); - // TODO Can we try harder to avoid collisions? - launcher.addLocalResource(destination, extraResource); - } - } - } - - // initialize addon pkg states for all componentInstanceStatus - Map pkgStatuses = new TreeMap<>(); - for (Metainfo appPkg : packageMetainfo.values()) { - // check each component of that addon to see if they apply to this - // component 'role' - for (ComponentsInAddonPackage comp : appPkg.getApplicationPackage() - .getComponents()) { - log.debug("Current component: {} component in metainfo: {}", roleName, - comp.getName()); - if (comp.getName().equals(roleGroup) - || comp.getName().equals(ADDON_FOR_ALL_COMPONENTS)) { - pkgStatuses.put(appPkg.getApplicationPackage().getName(), State.INIT); - } - } - } - log.debug("For component: {} pkg status map: {}", roleName, - pkgStatuses.toString()); - - // initialize the component instance state - getComponentStatuses().put(label, - new ComponentInstanceState( - roleGroup, - container.getId(), - getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME), - pkgStatuses)); - } - - @VisibleForTesting - protected void localizeConfigFile(ContainerLauncher launcher, - String roleName, String roleGroup, - ConfigFile configFile, - Map> configs, - MapOperations env, - SliderFileSystem fileSystem) - throws IOException { - ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType()); - providerUtils.localizeConfigFile(launcher, roleName, roleGroup, - configFile.getDictionaryName(), configFormat, configFile.getFileName(), - configs, env, fileSystem, getClusterName()); - } - - /** - * build the zookeeper registry path. - * - * @return the path the service registered at - * @throws NullPointerException if the service has not yet registered - */ - private String getZkRegistryPath() { - Preconditions.checkNotNull(yarnRegistry, "Yarn registry not bound"); - String path = yarnRegistry.getAbsoluteSelfRegistrationPath(); - Preconditions.checkNotNull(path, "Service record path not defined"); - return path; - } - - @Override - public void rebuildContainerDetails(List liveContainers, - String applicationId, Map providerRoleMap) { - for (Container container : liveContainers) { - // get the role name and label - ProviderRole role = providerRoleMap.get(ContainerPriority - .extractRole(container)); - if (role != null) { - String roleName = role.name; - String roleGroup = role.group; - String label = getContainerLabel(container, roleName, roleGroup); - log.info("Rebuilding in-memory: container {} in role {} in cluster {}", - container.getId(), roleName, applicationId); - getComponentStatuses().put(label, - new ComponentInstanceState(roleGroup, container.getId(), - applicationId)); - } else { - log.warn("Role not found for container {} in cluster {}", - container.getId(), applicationId); - } - } - } - - @Override - public boolean isSupportedRole(String role) { - return true; - } - - /** - * Handle registration calls from the agents - * - * @param registration registration entry - * - * @return response - */ - @Override - public RegistrationResponse handleRegistration(Register registration) { - log.info("Handling registration: {}", registration); - RegistrationResponse response = new RegistrationResponse(); - String label = registration.getLabel(); - String pkg = registration.getPkg(); - State agentState = registration.getActualState(); - String appVersion = registration.getAppVersion(); - - log.info("label: {} pkg: {}", label, pkg); - - if (getComponentStatuses().containsKey(label)) { - response.setResponseStatus(RegistrationStatus.OK); - ComponentInstanceState componentStatus = getComponentStatuses().get(label); - componentStatus.heartbeat(System.currentTimeMillis()); - updateComponentStatusWithAgentState(componentStatus, agentState); - - String roleName = getRoleName(label); - String roleGroup = getRoleGroup(label); - String containerId = getContainerId(label); - - if (SliderUtils.isSet(registration.getTags())) { - tags.recordAssignedTag(roleName, containerId, registration.getTags()); - } else { - response.setTags(tags.getTag(roleName, containerId)); - } - - String hostFqdn = registration.getPublicHostname(); - Map ports = registration.getAllocatedPorts(); - if (ports != null && !ports.isEmpty()) { - processAllocatedPorts(hostFqdn, roleName, roleGroup, containerId, ports); - } - - Map folders = registration.getLogFolders(); - if (folders != null && !folders.isEmpty()) { - publishFolderPaths(folders, containerId, roleName, hostFqdn); - } - - // Set app version if empty. It gets unset during upgrade - why? - checkAndSetContainerAppVersion(containerId, appVersion); - } else { - response.setResponseStatus(RegistrationStatus.FAILED); - response.setLog("Label not recognized."); - log.warn("Received registration request from unknown label {}", label); - } - log.info("Registration response: {}", response); - return response; - } - - // Checks if app version is empty. Sets it to the version as reported by the - // container during registration phase. - private void checkAndSetContainerAppVersion(String containerId, - String appVersion) { - StateAccessForProviders amState = getAmState(); - try { - RoleInstance role = amState.getOwnedContainer(containerId); - if (role != null) { - String currentAppVersion = role.appVersion; - log.debug("Container = {}, app version current = {} new = {}", - containerId, currentAppVersion, appVersion); - if (currentAppVersion == null - || currentAppVersion.equals(APP_VERSION_UNKNOWN)) { - amState.getOwnedContainer(containerId).appVersion = appVersion; - } - } - } catch (NoSuchNodeException e) { - // ignore - there is nothing to do if we don't find a container - log.warn("Owned container {} not found - {}", containerId, e); - } - } - - /** - * Handle heartbeat response from agents - * - * @param heartBeat incoming heartbeat from Agent - * - * @return response to send back - */ - @Override - public HeartBeatResponse handleHeartBeat(HeartBeat heartBeat) { - log.debug("Handling heartbeat: {}", heartBeat); - HeartBeatResponse response = new HeartBeatResponse(); - long id = heartBeat.getResponseId(); - response.setResponseId(id + 1L); - - String label = heartBeat.getHostname(); - String pkg = heartBeat.getPackage(); - - log.debug("package received: " + pkg); - - String roleName = getRoleName(label); - String roleGroup = getRoleGroup(label); - String containerId = getContainerId(label); - boolean doUpgrade = false; - if (isInUpgradeMode && upgradeContainers.contains(containerId)) { - doUpgrade = true; - } - - CommandScript cmdScript = getScriptPathForMasterPackage(roleGroup); - List commands = getApplicationComponent(roleGroup).getCommands(); - - if (!isDockerContainer(roleGroup) && !isYarnDockerContainer(roleGroup) - && (cmdScript == null || cmdScript.getScript() == null) - && commands.size() == 0) { - log.error( - "role.script is unavailable for {}. Commands will not be sent.", - roleName); - return response; - } - - String scriptPath = null; - long timeout = 600L; - if (cmdScript != null) { - scriptPath = cmdScript.getScript(); - timeout = cmdScript.getTimeout(); - } - - if (timeout == 0L) { - timeout = 600L; - } - - if (!getComponentStatuses().containsKey(label)) { - // container is completed but still heart-beating, send terminate signal - log.info( - "Sending terminate signal to completed container (still heartbeating): {}", - label); - response.setTerminateAgent(true); - return response; - } - - List statuses = heartBeat.getComponentStatus(); - if (statuses != null && !statuses.isEmpty()) { - log.info("status from agent: " + statuses.toString()); - for(ComponentStatus status : statuses){ - providerUtils.updateServiceRecord(getAmState(), yarnRegistry, - containerId, roleName, Collections.singletonList(status.getIp()), - status.getHostname()); - } - } - - Boolean isMaster = isMaster(roleGroup); - ComponentInstanceState componentStatus = getComponentStatuses().get(label); - componentStatus.heartbeat(System.currentTimeMillis()); - if (doUpgrade) { - switch (componentStatus.getState()) { - case STARTED: - componentStatus.setTargetState(State.UPGRADED); - break; - case UPGRADED: - componentStatus.setTargetState(State.STOPPED); - break; - case STOPPED: - componentStatus.setTargetState(State.TERMINATING); - break; - default: - break; - } - log.info("Current state = {} target state {}", - componentStatus.getState(), componentStatus.getTargetState()); - } - - if (appStopInitiated && !componentStatus.isStopInitiated()) { - log.info("Stop initiated for label {}", label); - componentStatus.setTargetState(State.STOPPED); - componentStatus.setStopInitiated(true); - } - - publishConfigAndExportGroups(heartBeat, componentStatus, roleGroup); - CommandResult result = null; - List reports = heartBeat.getReports(); - if (SliderUtils.isNotEmpty(reports)) { - CommandReport report = reports.get(0); - Map ports = report.getAllocatedPorts(); - if (SliderUtils.isNotEmpty(ports)) { - processAllocatedPorts(heartBeat.getFqdn(), roleName, roleGroup, containerId, ports); - } - result = CommandResult.getCommandResult(report.getStatus()); - Command command = Command.getCommand(report.getRoleCommand()); - componentStatus.applyCommandResult(result, command, pkg); - log.info("Component operation. Status: {}; new container state: {};" - + " new component state: {}", result, - componentStatus.getContainerState(), componentStatus.getState()); - - if (command == Command.INSTALL && SliderUtils.isNotEmpty(report.getFolders())) { - publishFolderPaths(report.getFolders(), containerId, roleName, heartBeat.getFqdn()); - } - } - - int waitForCount = getAmState().getInstanceDefinitionSnapshot(). - getAppConfOperations().getComponentOptInt(roleGroup, WAIT_HEARTBEAT, 0); - - if (id < waitForCount) { - log.info("Waiting until heartbeat count {}. Current val: {}", waitForCount, id); - getComponentStatuses().put(label, componentStatus); - return response; - } - - Command command = componentStatus.getNextCommand(doUpgrade); - try { - if (Command.NOP != command) { - log.debug("For comp {} pkg {} issuing {}", roleName, - componentStatus.getNextPkgToInstall(), command.toString()); - if (command == Command.INSTALL) { - log.info("Installing {} on {}.", roleName, containerId); - if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)){ - addInstallDockerCommand(roleName, roleGroup, containerId, - response, null, timeout); - } else if (scriptPath != null) { - addInstallCommand(roleName, roleGroup, containerId, response, - scriptPath, null, timeout, null); - } else { - // commands - ComponentCommand installCmd = null; - for (ComponentCommand compCmd : commands) { - if (compCmd.getName().equals("INSTALL")) { - installCmd = compCmd; - } - } - addInstallCommand(roleName, roleGroup, containerId, response, null, - installCmd, timeout, null); - } - componentStatus.commandIssued(command); - } else if (command == Command.INSTALL_ADDON) { - String nextPkgToInstall = componentStatus.getNextPkgToInstall(); - // retrieve scriptPath or command of that package for the component - for (ComponentsInAddonPackage comp : packageMetainfo - .get(nextPkgToInstall).getApplicationPackage().getComponents()) { - // given nextPkgToInstall and roleName is determined, the if below - // should only execute once per heartbeat - log.debug("Addon component: {} pkg: {} script: {}", comp.getName(), - nextPkgToInstall, comp.getCommandScript().getScript()); - if (comp.getName().equals(roleGroup) - || comp.getName().equals(ADDON_FOR_ALL_COMPONENTS)) { - scriptPath = comp.getCommandScript().getScript(); - if (scriptPath != null) { - addInstallCommand(roleName, roleGroup, containerId, response, - scriptPath, null, timeout, nextPkgToInstall); - } else { - ComponentCommand installCmd = null; - for (ComponentCommand compCmd : comp.getCommands()) { - if (compCmd.getName().equals("INSTALL")) { - installCmd = compCmd; - } - } - addInstallCommand(roleName, roleGroup, containerId, response, - null, installCmd, timeout, nextPkgToInstall); - } - } - } - componentStatus.commandIssued(command); - } else if (command == Command.START) { - // check against dependencies - boolean canExecute = commandOrder.canExecute(roleGroup, command, getComponentStatuses().values()); - if (canExecute) { - log.info("Starting {} on {}.", roleName, containerId); - if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)){ - addStartDockerCommand(roleName, roleGroup, containerId, - response, null, timeout, false); - } else if (scriptPath != null) { - addStartCommand(roleName, - roleGroup, - containerId, - response, - scriptPath, - null, - null, - timeout, - isMarkedAutoRestart(roleGroup)); - } else { - ComponentCommand startCmd = null; - for (ComponentCommand compCmd : commands) { - if (compCmd.getName().equals("START")) { - startCmd = compCmd; - } - } - ComponentCommand stopCmd = null; - for (ComponentCommand compCmd : commands) { - if (compCmd.getName().equals("STOP")) { - stopCmd = compCmd; - } - } - addStartCommand(roleName, roleGroup, containerId, response, null, - startCmd, stopCmd, timeout, false); - } - componentStatus.commandIssued(command); - } else { - log.info("Start of {} on {} delayed as dependencies have not started.", roleName, containerId); - } - } else if (command == Command.UPGRADE) { - addUpgradeCommand(roleName, roleGroup, containerId, response, - scriptPath, timeout); - componentStatus.commandIssued(command, true); - } else if (command == Command.STOP) { - log.info("Stop command being sent to container with id {}", - containerId); - addStopCommand(roleName, roleGroup, containerId, response, scriptPath, - timeout, doUpgrade); - componentStatus.commandIssued(command); - } else if (command == Command.TERMINATE) { - log.info("A formal terminate command is being sent to container {}" - + " in state {}", label, componentStatus.getState()); - response.setTerminateAgent(true); - } - } - - // if there is no outstanding command then retrieve config - if (isMaster && componentStatus.getState() == State.STARTED - && command == Command.NOP) { - if (!componentStatus.getConfigReported()) { - log.info("Requesting applied config for {} on {}.", roleName, containerId); - if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)){ - addGetConfigDockerCommand(roleName, roleGroup, containerId, response); - } else { - addGetConfigCommand(roleName, roleGroup, containerId, response); - } - } - } - - // if restart is required then signal - response.setRestartEnabled(false); - if (componentStatus.getState() == State.STARTED - && command == Command.NOP && isMarkedAutoRestart(roleGroup)) { - response.setRestartEnabled(true); - } - - //If INSTALL_FAILED and no INSTALL is scheduled let the agent fail - if (componentStatus.getState() == State.INSTALL_FAILED - && command == Command.NOP) { - log.warn("Sending terminate signal to container that failed installation: {}", label); - response.setTerminateAgent(true); - } - - } catch (SliderException e) { - log.warn("Component instance failed operation.", e); - componentStatus.applyCommandResult(CommandResult.FAILED, command, null); - } - - log.debug("Heartbeat response: " + response); - return response; - } - - private boolean isDockerContainer(String roleGroup) { - String type = getApplicationComponent(roleGroup).getType(); - if (SliderUtils.isSet(type)) { - return type.toLowerCase().equals(SliderUtils.DOCKER) || type.toLowerCase().equals(SliderUtils.DOCKER_YARN); - } - return false; - } - - private boolean isYarnDockerContainer(String roleGroup) { - String type = getApplicationComponent(roleGroup).getType(); - if (SliderUtils.isSet(type)) { - return type.toLowerCase().equals(SliderUtils.DOCKER_YARN); - } - return false; - } - - protected void processAllocatedPorts(String fqdn, - String roleName, - String roleGroup, - String containerId, - Map ports) { - RoleInstance instance; - try { - instance = getAmState().getOwnedContainer(containerId); - } catch (NoSuchNodeException e) { - log.warn("Failed to locate instance of container {}", containerId, e); - instance = null; - } - for (Map.Entry port : ports.entrySet()) { - String portname = port.getKey(); - String portNo = port.getValue(); - log.info("Recording allocated port for {} as {}", portname, portNo); - - // add the allocated ports to the global list as well as per container list - // per container allocation will over-write each other in the global - this.getAllocatedPorts().put(portname, portNo); - this.getAllocatedPorts(containerId).put(portname, portNo); - if (instance != null) { - try { - // if the returned value is not a single port number then there are no - // meaningful way for Slider to use it during export - // No need to error out as it may not be the responsibility of the component - // to allocate port or the component may need an array of ports - instance.registerPortEndpoint(Integer.valueOf(portNo), portname); - } catch (NumberFormatException e) { - log.warn("Failed to parse {}", portNo, e); - } - } - } - - processAndPublishComponentSpecificData(ports, containerId, fqdn, roleGroup); - processAndPublishComponentSpecificExports(ports, containerId, fqdn, roleName, roleGroup); - - // and update registration entries - if (instance != null) { - queueAccess.put(new RegisterComponentInstance(instance.getId(), - roleName, roleGroup, 0, TimeUnit.MILLISECONDS)); - } - } - - private void updateComponentStatusWithAgentState( - ComponentInstanceState componentStatus, State agentState) { - if (agentState != null) { - componentStatus.setState(agentState); - } - } - - @Override - public Map buildMonitorDetails(ClusterDescription clusterDesc) { - Map details = super.buildMonitorDetails(clusterDesc); - buildRoleHostDetails(details); - return details; - } - - public void applyInitialRegistryDefinitions(URL amWebURI, - URL agentOpsURI, - URL agentStatusURI, - ServiceRecord serviceRecord) - throws IOException { - super.applyInitialRegistryDefinitions(amWebURI, - serviceRecord); - - try { - URL restURL = new URL(agentOpsURI, SLIDER_PATH_AGENTS); - URL agentStatusURL = new URL(agentStatusURI, SLIDER_PATH_AGENTS); - - serviceRecord.addInternalEndpoint( - new Endpoint(CustomRegistryConstants.AGENT_SECURE_REST_API, - ProtocolTypes.PROTOCOL_REST, - restURL.toURI())); - serviceRecord.addInternalEndpoint( - new Endpoint(CustomRegistryConstants.AGENT_ONEWAY_REST_API, - ProtocolTypes.PROTOCOL_REST, - agentStatusURL.toURI())); - } catch (URISyntaxException e) { - throw new IOException(e); - } - - // identify client component - Component client = null; - for (Component component : getMetaInfo().getApplication().getComponents()) { - if (component.getCategory().equals("CLIENT")) { - client = component; - break; - } - } - if (client == null) { - log.info("No client component specified, not publishing client configs"); - return; - } - - // register AM-generated client configs - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - MapOperations clientOperations = appConf.getOrAddComponent(client.getName()); - appConf.resolve(); - if (!clientOperations.getOptionBool(AM_CONFIG_GENERATION, - false)) { - log.info("AM config generation is false, not publishing client configs"); - return; - } - - // build and localize configuration files - Map> configurations = new TreeMap<>(); - Map tokens = providerUtils.getStandardTokenMap(appConf, - getAmState().getInternalsSnapshot(), client.getName(), - client.getName(), getClusterName()); - - for (ConfigFile configFile : getMetaInfo().getComponentConfigFiles(client.getName())) { - addNamedConfiguration(configFile.getDictionaryName(), - appConf.getGlobalOptions().options, configurations, tokens, null, - client.getName(), client.getName()); - if (appConf.getComponent(client.getName()) != null) { - addNamedConfiguration(configFile.getDictionaryName(), - appConf.getComponent(client.getName()).options, configurations, - tokens, null, client.getName(), client.getName()); - } - } - - //do a final replacement of re-used configs - dereferenceAllConfigs(configurations); - - for (ConfigFile configFile : getMetaInfo().getComponentConfigFiles(client.getName())) { - ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType()); - - Map config = configurations.get(configFile.getDictionaryName()); - ConfigUtils.prepConfigForTemplateOutputter(configFormat, config, - fileSystem, getClusterName(), - new File(configFile.getFileName()).getName()); - PublishedConfiguration publishedConfiguration = - new PublishedConfiguration(configFile.getDictionaryName(), - config.entrySet()); - getAmState().getPublishedSliderConfigurations().put( - configFile.getDictionaryName(), publishedConfiguration); - log.info("Publishing AM configuration {}", configFile.getDictionaryName()); - } - } - - @Override - public void notifyContainerCompleted(ContainerId containerId) { - // containers get allocated and free'ed without being assigned to any - // component - so many of the data structures may not be initialized - if (containerId != null) { - String containerIdStr = containerId.toString(); - if (getComponentInstanceData().containsKey(containerIdStr)) { - getComponentInstanceData().remove(containerIdStr); - log.info("Removing container specific data for {}", containerIdStr); - publishComponentInstanceData(); - } - - if (this.allocatedPorts.containsKey(containerIdStr)) { - Map portsByContainerId = getAllocatedPorts(containerIdStr); - this.allocatedPorts.remove(containerIdStr); - // free up the allocations from global as well - // if multiple containers allocate global ports then last one - // wins and similarly first one removes it - its not supported anyway - for(String portName : portsByContainerId.keySet()) { - getAllocatedPorts().remove(portName); - } - - } - - String componentName = null; - synchronized (this.componentStatuses) { - for (String label : getComponentStatuses().keySet()) { - if (label.startsWith(containerIdStr)) { - componentName = getRoleName(label); - log.info("Removing component status for label {}", label); - getComponentStatuses().remove(label); - } - } - } - - tags.releaseTag(componentName, containerIdStr); - - synchronized (this.containerExportsMap) { - Set containerExportSets = containerExportsMap.get(containerIdStr); - if (containerExportSets != null) { - for (String containerExportStr : containerExportSets) { - String[] parts = containerExportStr.split(":"); - Map> exportGroup = getCurrentExports(parts[0]); - List exports = exportGroup.get(parts[1]); - List exportToRemove = new ArrayList(); - for (ExportEntry export : exports) { - if (containerIdStr.equals(export.getContainerId())) { - exportToRemove.add(export); - } - } - exports.removeAll(exportToRemove); - } - log.info("Removing container exports for {}", containerIdStr); - containerExportsMap.remove(containerIdStr); - } - } - } - } - - /** - * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default. - * - * @param instanceDefinition - */ - private void readAndSetHeartbeatMonitoringInterval(AggregateConf instanceDefinition) { - String hbMonitorInterval = instanceDefinition.getAppConfOperations(). - getGlobalOptions().getOption(HEARTBEAT_MONITOR_INTERVAL, - Integer.toString(DEFAULT_HEARTBEAT_MONITOR_INTERVAL)); - try { - setHeartbeatMonitorInterval(Integer.parseInt(hbMonitorInterval)); - } catch (NumberFormatException e) { - log.warn( - "Bad value {} for {}. Defaulting to ", - hbMonitorInterval, - HEARTBEAT_MONITOR_INTERVAL, - DEFAULT_HEARTBEAT_MONITOR_INTERVAL); - } - } - - /** - * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default. - * - * @param instanceDefinition - */ - private void initializeAgentDebugCommands(AggregateConf instanceDefinition) { - String launchParameterStr = instanceDefinition.getAppConfOperations(). - getGlobalOptions().getOption(AGENT_INSTANCE_DEBUG_DATA, ""); - agentLaunchParameter = new AgentLaunchParameter(launchParameterStr); - } - - @VisibleForTesting - protected Map getLogFolderExports() { - return logFolderExports; - } - - @VisibleForTesting - protected Map getWorkFolderExports() { - return workFolderExports; - } - - protected Metainfo getMetaInfo() { - return getMetaInfo(null); - } - - @VisibleForTesting - protected Metainfo getMetaInfo(String roleGroup) { - String mapKey = DEFAULT_METAINFO_MAP_KEY; - if (roleGroup != null) { - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - mapKey = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, - DEFAULT_METAINFO_MAP_KEY); - } - MetainfoHolder mh = this.metaInfoMap.get(mapKey); - if (mh == null) { - return null; - } - return mh.metaInfo; - } - - @VisibleForTesting - protected Map getComponentStatuses() { - return componentStatuses; - } - - @VisibleForTesting - protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, - String appDef, boolean addonPackage) throws IOException, - BadConfigException { - return AgentUtils.getApplicationMetainfo(fileSystem, appDef, addonPackage); - } - - @VisibleForTesting - protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, - String appDef) throws IOException, BadConfigException { - return getApplicationMetainfo(fileSystem, appDef, false); - } - - @VisibleForTesting - protected void setHeartbeatMonitorInterval(int heartbeatMonitorInterval) { - this.heartbeatMonitorInterval = heartbeatMonitorInterval; - } - - public void setInUpgradeMode(boolean inUpgradeMode) { - this.isInUpgradeMode = inUpgradeMode; - } - - public void addUpgradeContainers(Set upgradeContainers) { - this.upgradeContainers.addAll(upgradeContainers); - } - - public void setAppStopInitiated(boolean appStopInitiated) { - this.appStopInitiated = appStopInitiated; - } - - /** - * Read all default configs - * - * @param fileSystem fs - * @param appDef app default path - * @param metainfo metadata - * - * @return configuration maps - * - * @throws IOException - */ - protected Map initializeDefaultConfigs(SliderFileSystem fileSystem, - String appDef, Metainfo metainfo) throws IOException { - Map defaultConfigMap = new HashMap<>(); - if (SliderUtils.isNotEmpty(metainfo.getApplication().getConfigFiles())) { - for (ConfigFile configFile : metainfo.getApplication().getConfigFiles()) { - DefaultConfig config = null; - try { - config = AgentUtils.getDefaultConfig(fileSystem, appDef, configFile.getDictionaryName() + ".xml"); - } catch (IOException e) { - log.warn("Default config file not found. Only the config as input during create will be applied for {}", - configFile.getDictionaryName()); - } - if (config != null) { - defaultConfigMap.put(configFile.getDictionaryName(), config); - } - } - } - - return defaultConfigMap; - } - - protected Map getDefaultConfigs(String roleGroup) { - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - String mapKey = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, - DEFAULT_METAINFO_MAP_KEY); - return metaInfoMap.get(mapKey).defaultConfigs; - } - - private int getHeartbeatMonitorInterval() { - return this.heartbeatMonitorInterval; - } - - private String getClusterName() { - if (SliderUtils.isUnset(clusterName)) { - clusterName = getAmState().getInternalsSnapshot().get(OptionKeys.APPLICATION_NAME); - } - return clusterName; - } - - @VisibleForTesting - protected void publishApplicationInstanceData(String name, String description, - Iterable> entries) { - providerUtils.publishApplicationInstanceData(name, description, entries, - getAmState()); - } - - /** - * Get a list of all hosts for all role/container per role - * - * @return the map of role->node - */ - protected Map> getRoleClusterNodeMapping() { - return amState.getRoleClusterNodeMapping(); - } - - private String getContainerLabel(Container container, String role, String group) { - if (role.equals(group)) { - return container.getId().toString() + LABEL_MAKER + role; - } else { - return container.getId().toString() + LABEL_MAKER + role + LABEL_MAKER + - group; - } - } - - protected String getClusterInfoPropertyValue(String name) { - StateAccessForProviders accessor = getAmState(); - assert accessor.isApplicationLive(); - ClusterDescription description = accessor.getClusterStatus(); - return description.getInfo(name); - } - - protected String getClusterOptionPropertyValue(String name) - throws BadConfigException { - StateAccessForProviders accessor = getAmState(); - assert accessor.isApplicationLive(); - ClusterDescription description = accessor.getClusterStatus(); - return description.getMandatoryOption(name); - } - - /** - * Lost heartbeat from the container - release it and ask for a replacement (async operation) - * - * @param label - * @param containerId - */ - protected void lostContainer( - String label, - ContainerId containerId) { - getComponentStatuses().remove(label); - getQueueAccess().put(new ProviderReportedContainerLoss(containerId)); - } - - /** - * Build the provider status, can be empty - * - * @return the provider status - map of entries to add to the info section - */ - public Map buildProviderStatus() { - Map stats = new HashMap(); - return stats; - } - - @VisibleForTesting - protected void publishFolderPaths( - Map folders, String containerId, String componentName, String hostFqdn) { - providerUtils.publishFolderPaths(folders, containerId, componentName, hostFqdn, - getAmState(), getLogFolderExports(), getWorkFolderExports()); - } - - /** - * Process return status for component instances - * - * @param heartBeat - * @param componentStatus - */ - protected void publishConfigAndExportGroups(HeartBeat heartBeat, - ComponentInstanceState componentStatus, String componentGroup) { - List statuses = heartBeat.getComponentStatus(); - if (statuses != null && !statuses.isEmpty()) { - log.info("Processing {} status reports.", statuses.size()); - for (ComponentStatus status : statuses) { - log.info("Status report: {}", status.toString()); - - if (status.getConfigs() != null) { - Application application = getMetaInfo(componentGroup).getApplication(); - - if ((!canAnyMasterPublishConfig(componentGroup) || canPublishConfig(componentGroup)) && - !getAmState().getAppConfSnapshot().getComponentOptBool( - componentGroup, AM_CONFIG_GENERATION, false)) { - // If no Master can explicitly publish then publish if its a master - // Otherwise, wait till the master that can publish is ready - - Set exportedConfigs = new HashSet(); - String exportedConfigsStr = application.getExportedConfigs(); - boolean exportedAllConfigs = exportedConfigsStr == null || exportedConfigsStr.isEmpty(); - if (!exportedAllConfigs) { - for (String exportedConfig : exportedConfigsStr.split(",")) { - if (exportedConfig.trim().length() > 0) { - exportedConfigs.add(exportedConfig.trim()); - } - } - } - - for (String key : status.getConfigs().keySet()) { - if ((!exportedAllConfigs && exportedConfigs.contains(key)) || - exportedAllConfigs) { - Map configs = status.getConfigs().get(key); - publishApplicationInstanceData(key, key, configs.entrySet()); - } - } - } - - List appExportGroups = application.getExportGroups(); - boolean hasExportGroups = SliderUtils.isNotEmpty(appExportGroups); - - Set appExports = new HashSet(); - String appExportsStr = getApplicationComponent(componentGroup).getAppExports(); - if (SliderUtils.isSet(appExportsStr)) { - for (String appExport : appExportsStr.split(",")) { - if (!appExport.trim().isEmpty()) { - appExports.add(appExport.trim()); - } - } - } - - if (hasExportGroups && !appExports.isEmpty()) { - String configKeyFormat = "${site.%s.%s}"; - String hostKeyFormat = "${%s_HOST}"; - - // publish export groups if any - Map replaceTokens = new HashMap(); - for (Map.Entry> entry : getRoleClusterNodeMapping().entrySet()) { - String hostName = providerUtils.getHostsList( - entry.getValue().values(), true).iterator().next(); - replaceTokens.put(String.format(hostKeyFormat, entry.getKey().toUpperCase(Locale.ENGLISH)), hostName); - } - - for (String key : status.getConfigs().keySet()) { - Map configs = status.getConfigs().get(key); - for (String configKey : configs.keySet()) { - String lookupKey = String.format(configKeyFormat, key, configKey); - replaceTokens.put(lookupKey, configs.get(configKey)); - } - } - - Set modifiedGroups = new HashSet(); - for (ExportGroup exportGroup : appExportGroups) { - List exports = exportGroup.getExports(); - if (SliderUtils.isNotEmpty(exports)) { - String exportGroupName = exportGroup.getName(); - ConcurrentHashMap> map = - (ConcurrentHashMap>)getCurrentExports(exportGroupName); - for (Export export : exports) { - if (canBeExported(exportGroupName, export.getName(), appExports)) { - String value = export.getValue(); - // replace host names - for (String token : replaceTokens.keySet()) { - if (value.contains(token)) { - value = value.replace(token, replaceTokens.get(token)); - } - } - ExportEntry entry = new ExportEntry(); - entry.setLevel(APPLICATION_TAG); - entry.setValue(value); - entry.setUpdatedTime(new Date().toString()); - // over-write, app exports are singletons - map.put(export.getName(), new ArrayList(Arrays.asList(entry))); - log.info("Preparing to publish. Key {} and Value {}", export.getName(), value); - } - } - modifiedGroups.add(exportGroupName); - } - } - publishModifiedExportGroups(modifiedGroups); - } - - log.info("Received and processed config for {}", heartBeat.getHostname()); - componentStatus.setConfigReported(true); - - } - } - } - } - - private boolean canBeExported(String exportGroupName, String name, Set appExports) { - return appExports.contains(String.format("%s-%s", exportGroupName, name)); - } - - protected Map> getCurrentExports(String groupName) { - if (!this.exportGroups.containsKey(groupName)) { - synchronized (this.exportGroups) { - if (!this.exportGroups.containsKey(groupName)) { - this.exportGroups.put(groupName, new ConcurrentHashMap>()); - } - } - } - - return this.exportGroups.get(groupName); - } - - private void publishModifiedExportGroups(Set modifiedGroups) { - for (String roleGroup : modifiedGroups) { - Map> entries = this.exportGroups.get(roleGroup); - // Publish in old format for the time being - Map simpleEntries = new HashMap(); - for (Entry> entry : entries.entrySet()) { - List exports = entry.getValue(); - if (SliderUtils.isNotEmpty(exports)) { - // there is no support for multiple exports per name - so extract only the first one - simpleEntries.put(entry.getKey(), entry.getValue().get(0).getValue()); - } - } - publishApplicationInstanceData(roleGroup, roleGroup, - simpleEntries.entrySet()); - - PublishedExports exports = new PublishedExports(roleGroup); - exports.setUpdated(new Date().getTime()); - exports.putValues(entries.entrySet()); - getAmState().getPublishedExportsSet().put(roleGroup, exports); - } - } - - /** Publish component instance specific data if the component demands it */ - protected void processAndPublishComponentSpecificData(Map ports, - String containerId, - String hostFqdn, - String componentGroup) { - String portVarFormat = "${site.%s}"; - String hostNamePattern = "${THIS_HOST}"; - Map toPublish = new HashMap(); - - Application application = getMetaInfo(componentGroup).getApplication(); - for (Component component : application.getComponents()) { - if (component.getName().equals(componentGroup)) { - if (component.getComponentExports().size() > 0) { - - for (ComponentExport export : component.getComponentExports()) { - String templateToExport = export.getValue(); - for (String portName : ports.keySet()) { - boolean publishData = false; - String portValPattern = String.format(portVarFormat, portName); - if (templateToExport.contains(portValPattern)) { - templateToExport = templateToExport.replace(portValPattern, ports.get(portName)); - publishData = true; - } - if (templateToExport.contains(hostNamePattern)) { - templateToExport = templateToExport.replace(hostNamePattern, hostFqdn); - publishData = true; - } - if (publishData) { - toPublish.put(export.getName(), templateToExport); - log.info("Publishing {} for name {} and container {}", - templateToExport, export.getName(), containerId); - } - } - } - } - } - } - - if (toPublish.size() > 0) { - Map perContainerData = null; - if (!getComponentInstanceData().containsKey(containerId)) { - perContainerData = new ConcurrentHashMap(); - } else { - perContainerData = getComponentInstanceData().get(containerId); - } - perContainerData.putAll(toPublish); - getComponentInstanceData().put(containerId, perContainerData); - publishComponentInstanceData(); - } - } - - /** Publish component instance specific data if the component demands it */ - protected void processAndPublishComponentSpecificExports(Map ports, - String containerId, - String hostFqdn, - String compName, - String compGroup) { - String portVarFormat = "${site.%s}"; - String hostNamePattern = "${" + compGroup + "_HOST}"; - - List appExportGroups = getMetaInfo(compGroup).getApplication().getExportGroups(); - Component component = getApplicationComponent(compGroup); - if (component != null && SliderUtils.isSet(component.getCompExports()) - && SliderUtils.isNotEmpty(appExportGroups)) { - - Set compExports = new HashSet(); - String compExportsStr = component.getCompExports(); - for (String compExport : compExportsStr.split(",")) { - if (!compExport.trim().isEmpty()) { - compExports.add(compExport.trim()); - } - } - - Date now = new Date(); - Set modifiedGroups = new HashSet(); - for (ExportGroup exportGroup : appExportGroups) { - List exports = exportGroup.getExports(); - if (SliderUtils.isNotEmpty(exports)) { - String exportGroupName = exportGroup.getName(); - ConcurrentHashMap> map = - (ConcurrentHashMap>) getCurrentExports(exportGroupName); - for (Export export : exports) { - if (canBeExported(exportGroupName, export.getName(), compExports)) { - log.info("Attempting to publish {} of group {} for component type {}", - export.getName(), exportGroupName, compName); - String templateToExport = export.getValue(); - for (String portName : ports.keySet()) { - boolean publishData = false; - String portValPattern = String.format(portVarFormat, portName); - if (templateToExport.contains(portValPattern)) { - templateToExport = templateToExport.replace(portValPattern, ports.get(portName)); - publishData = true; - } - if (templateToExport.contains(hostNamePattern)) { - templateToExport = templateToExport.replace(hostNamePattern, hostFqdn); - publishData = true; - } - if (publishData) { - ExportEntry entryToAdd = new ExportEntry(); - entryToAdd.setLevel(COMPONENT_TAG); - entryToAdd.setValue(templateToExport); - entryToAdd.setUpdatedTime(now.toString()); - entryToAdd.setContainerId(containerId); - entryToAdd.setTag(tags.getTag(compName, containerId)); - - List existingList = - map.putIfAbsent(export.getName(), new CopyOnWriteArrayList(Arrays.asList(entryToAdd))); - - // in-place edit, no lock needed - if (existingList != null) { - boolean updatedInPlace = false; - for (ExportEntry entry : existingList) { - if (containerId.toLowerCase(Locale.ENGLISH) - .equals(entry.getContainerId())) { - entryToAdd.setValue(templateToExport); - entryToAdd.setUpdatedTime(now.toString()); - updatedInPlace = true; - } - } - if (!updatedInPlace) { - existingList.add(entryToAdd); - } - } - - log.info("Publishing {} for name {} and container {}", - templateToExport, export.getName(), containerId); - modifiedGroups.add(exportGroupName); - synchronized (containerExportsMap) { - if (!containerExportsMap.containsKey(containerId)) { - containerExportsMap.put(containerId, new HashSet()); - } - Set containerExportMaps = containerExportsMap.get(containerId); - containerExportMaps.add(String.format("%s:%s", exportGroupName, export.getName())); - } - } - } - } - } - } - } - publishModifiedExportGroups(modifiedGroups); - } - } - - private void publishComponentInstanceData() { - Map dataToPublish = new HashMap(); - for (String container : getComponentInstanceData().keySet()) { - for (String prop : getComponentInstanceData().get(container).keySet()) { - dataToPublish.put( - container + "." + prop, getComponentInstanceData().get(container).get(prop)); - } - } - publishApplicationInstanceData(COMPONENT_DATA_TAG, COMPONENT_DATA_TAG, dataToPublish.entrySet()); - } - - /** - * Return Component based on group - * - * @param roleGroup component group - * - * @return the component entry or null for no match - */ - protected Component getApplicationComponent(String roleGroup) { - Metainfo metainfo = getMetaInfo(roleGroup); - if (metainfo == null) { - return null; - } - return metainfo.getApplicationComponent(roleGroup); - } - - /** - * Extract script path from the application metainfo - * - * @param roleGroup component group - * @return the script path or null for no match - */ - protected CommandScript getScriptPathForMasterPackage(String roleGroup) { - Component component = getApplicationComponent(roleGroup); - if (component != null) { - return component.getCommandScript(); - } - return null; - } - - /** - * Is the role of type MASTER - * - * @param roleGroup component group - * - * @return true if the role category is MASTER - */ - protected boolean isMaster(String roleGroup) { - Component component = getApplicationComponent(roleGroup); - if (component != null) { - if (component.getCategory().equals("MASTER")) { - return true; - } - } - return false; - } - - /** - * Can the role publish configuration - * - * @param roleGroup component group - * - * @return true if it can be pubished - */ - protected boolean canPublishConfig(String roleGroup) { - Component component = getApplicationComponent(roleGroup); - if (component != null) { - return Boolean.TRUE.toString().equals(component.getPublishConfig()); - } - return false; - } - - /** - * Checks if the role is marked auto-restart - * - * @param roleGroup component group - * - * @return true if it is auto-restart - */ - protected boolean isMarkedAutoRestart(String roleGroup) { - Component component = getApplicationComponent(roleGroup); - if (component != null) { - return component.getAutoStartOnFailureBoolean(); - } - return false; - } - - /** - * Can any master publish config explicitly, if not a random master is used - * - * @return true if the condition holds - */ - protected boolean canAnyMasterPublishConfig(String roleGroup) { - if (canAnyMasterPublish == null) { - Application application = getMetaInfo(roleGroup).getApplication(); - if (application == null) { - log.error("Malformed app definition: Expect application as root element in the metainfo.xml"); - } else { - for (Component component : application.getComponents()) { - if (Boolean.TRUE.toString().equals(component.getPublishConfig()) && - component.getCategory().equals("MASTER")) { - canAnyMasterPublish = true; - } - } - } - } - - if (canAnyMasterPublish == null) { - canAnyMasterPublish = false; - } - return canAnyMasterPublish; - } - - private String getRoleName(String label) { - int index1 = label.indexOf(LABEL_MAKER); - int index2 = label.lastIndexOf(LABEL_MAKER); - if (index1 == index2) { - return label.substring(index1 + LABEL_MAKER.length()); - } else { - return label.substring(index1 + LABEL_MAKER.length(), index2); - } - } - - private String getRoleGroup(String label) { - return label.substring(label.lastIndexOf(LABEL_MAKER) + LABEL_MAKER.length()); - } - - private String getContainerId(String label) { - return label.substring(0, label.indexOf(LABEL_MAKER)); - } - - /** - * Add install command to the heartbeat response - * - * @param roleName - * @param roleGroup - * @param containerId - * @param response - * @param scriptPath - * @param pkg - * when this field is null, it indicates the command is for the - * master package; while not null, for the package named by this - * field - * @throws SliderException - */ - @VisibleForTesting - protected void addInstallCommand(String roleName, - String roleGroup, - String containerId, - HeartBeatResponse response, - String scriptPath, - ComponentCommand compCmd, - long timeout, - String pkg) - throws SliderException { - assert getAmState().isApplicationLive(); - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - - ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND); - prepareExecutionCommand(cmd); - String clusterName = getClusterName(); - cmd.setClusterName(clusterName); - cmd.setRoleCommand(Command.INSTALL.toString()); - cmd.setServiceName(clusterName); - cmd.setComponentName(roleName); - cmd.setRole(roleName); - cmd.setPkg(pkg); - Map hostLevelParams = new TreeMap(); - hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir())); - hostLevelParams.put(PACKAGE_LIST, getPackageList(roleGroup)); - hostLevelParams.put(CONTAINER_ID, containerId); - cmd.setHostLevelParams(hostLevelParams); - - Map> configurations = - buildCommandConfigurations(appConf, getAmState().getInternalsSnapshot(), - containerId, roleName, roleGroup); - cmd.setConfigurations(configurations); - Map> componentConfigurations = buildComponentConfigurations(appConf); - cmd.setComponentConfigurations(componentConfigurations); - - if (SliderUtils.isSet(scriptPath)) { - cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false)); - } else { - // assume it to be default shell command - ComponentCommand effectiveCommand = compCmd; - if (effectiveCommand == null) { - effectiveCommand = ComponentCommand.getDefaultComponentCommand("INSTALL"); - } - cmd.setCommandParams(commandParametersSet(effectiveCommand, timeout, false)); - configurations.get("global").put("exec_cmd", effectiveCommand.getExec()); - } - - cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME)); - - response.addExecutionCommand(cmd); - - log.debug("command looks like: {} ", cmd); - } - - @VisibleForTesting - protected void addInstallDockerCommand(String roleName, - String roleGroup, - String containerId, - HeartBeatResponse response, - ComponentCommand compCmd, - long timeout) - throws SliderException { - assert getAmState().isApplicationLive(); - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - - ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND); - prepareExecutionCommand(cmd); - String clusterName = getClusterName(); - cmd.setClusterName(clusterName); - cmd.setRoleCommand(Command.INSTALL.toString()); - cmd.setServiceName(clusterName); - cmd.setComponentName(roleName); - cmd.setRole(roleName); - Map hostLevelParams = new TreeMap(); - hostLevelParams.put(PACKAGE_LIST, getPackageList(roleGroup)); - hostLevelParams.put(CONTAINER_ID, containerId); - cmd.setHostLevelParams(hostLevelParams); - - Map> configurations = buildCommandConfigurations( - appConf, getAmState().getInternalsSnapshot(), containerId, roleName, - roleGroup); - cmd.setConfigurations(configurations); - Map> componentConfigurations = buildComponentConfigurations(appConf); - cmd.setComponentConfigurations(componentConfigurations); - - ComponentCommand effectiveCommand = compCmd; - if (compCmd == null) { - effectiveCommand = new ComponentCommand(); - effectiveCommand.setName("INSTALL"); - effectiveCommand.setExec("DEFAULT"); - } - cmd.setCommandParams(setCommandParameters(effectiveCommand, timeout, false)); - configurations.get("global").put("exec_cmd", effectiveCommand.getExec()); - - cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME)); - cmd.addContainerDetails(roleGroup, getMetaInfo(roleGroup)); - - Map dockerConfig = new HashMap(); - if(isYarnDockerContainer(roleGroup)){ - //put nothing - cmd.setYarnDockerMode(true); - } else { - dockerConfig.put( - "docker.command_path", - getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, - "commandPath")); - dockerConfig.put("docker.image_name", - getConfigFromMetaInfo(roleGroup, "image")); - } - configurations.put("docker", dockerConfig); - - log.debug("Docker- command: {}", cmd.toString()); - - response.addExecutionCommand(cmd); - } - - private Map setCommandParameters(String scriptPath, - long timeout, boolean recordConfig) { - Map cmdParams = new TreeMap(); - cmdParams.put("service_package_folder", - "${AGENT_WORK_ROOT}/work/app/definition/package"); - cmdParams.put("script", scriptPath); - cmdParams.put("schema_version", "2.0"); - cmdParams.put("command_timeout", Long.toString(timeout)); - cmdParams.put("script_type", AbstractComponent.TYPE_PYTHON); - cmdParams.put("record_config", Boolean.toString(recordConfig)); - return cmdParams; - } - - private Map setCommandParameters(ComponentCommand compCmd, - long timeout, boolean recordConfig) { - Map cmdParams = new TreeMap(); - cmdParams.put("service_package_folder", - "${AGENT_WORK_ROOT}/work/app/definition/package"); - cmdParams.put("command", compCmd.getExec()); - cmdParams.put("schema_version", "2.0"); - cmdParams.put("command_timeout", Long.toString(timeout)); - cmdParams.put("script_type", compCmd.getType()); - cmdParams.put("record_config", Boolean.toString(recordConfig)); - return cmdParams; - } - - private Map> buildComponentConfigurations( - ConfTreeOperations appConf) { - return appConf.getComponents(); - } - - protected static String getPackageListFromApplication(Application application) { - String pkgFormatString = "{\"type\":\"%s\",\"name\":\"%s\"}"; - String pkgListFormatString = "[%s]"; - List packages = new ArrayList<>(); - if (application != null) { - if (application.getPackages().size() > 0) { - // no-op if there are packages that are not OS-specific, as these - // will be localized by AM rather than the Agent - // this should be backwards compatible, as there was previously an - // XML parsing bug that ensured non-OS-specific packages did not exist - } else { - List osSpecifics = application.getOSSpecifics(); - if (osSpecifics != null && osSpecifics.size() > 0) { - for (OSSpecific osSpecific : osSpecifics) { - if (osSpecific.getOsType().equals("any")) { - for (OSPackage osPackage : osSpecific.getPackages()) { - packages.add(String.format(pkgFormatString, osPackage.getType(), osPackage.getName())); - } - } - } - } - } - } - - if (!packages.isEmpty()) { - return "[" + SliderUtils.join(packages, ",", false) + "]"; - } else { - return "[]"; - } - } - - private String getPackageList(String roleGroup) { - return getPackageListFromApplication(getMetaInfo(roleGroup).getApplication()); - } - - private void prepareExecutionCommand(ExecutionCommand cmd) { - cmd.setTaskId(taskId.incrementAndGet()); - cmd.setCommandId(cmd.getTaskId() + "-1"); - } - - private Map commandParametersSet(String scriptPath, long timeout, boolean recordConfig) { - Map cmdParams = new TreeMap<>(); - cmdParams.put("service_package_folder", - "${AGENT_WORK_ROOT}/work/app/definition/package"); - cmdParams.put("script", scriptPath); - cmdParams.put("schema_version", "2.0"); - cmdParams.put("command_timeout", Long.toString(timeout)); - cmdParams.put("script_type", "PYTHON"); - cmdParams.put("record_config", Boolean.toString(recordConfig)); - return cmdParams; - } - - private Map commandParametersSet(ComponentCommand compCmd, long timeout, boolean recordConfig) { - Map cmdParams = new TreeMap<>(); - cmdParams.put("service_package_folder", - "${AGENT_WORK_ROOT}/work/app/definition/package"); - cmdParams.put("command", compCmd.getExec()); - cmdParams.put("schema_version", "2.0"); - cmdParams.put("command_timeout", Long.toString(timeout)); - cmdParams.put("script_type", compCmd.getType()); - cmdParams.put("record_config", Boolean.toString(recordConfig)); - return cmdParams; - } - - @VisibleForTesting - protected void addStatusCommand(String roleName, - String roleGroup, - String containerId, - HeartBeatResponse response, - String scriptPath, - long timeout) - throws SliderException { - assert getAmState().isApplicationLive(); - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)) { - addStatusDockerCommand(roleName, roleGroup, containerId, response, - scriptPath, timeout); - return; - } - - StatusCommand cmd = new StatusCommand(); - String clusterName = getClusterName(); - - cmd.setCommandType(AgentCommandType.STATUS_COMMAND); - cmd.setComponentName(roleName); - cmd.setServiceName(clusterName); - cmd.setClusterName(clusterName); - cmd.setRoleCommand(StatusCommand.STATUS_COMMAND); - - Map hostLevelParams = new TreeMap(); - hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir())); - hostLevelParams.put(CONTAINER_ID, containerId); - cmd.setHostLevelParams(hostLevelParams); - - cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false)); - - Map> configurations = - buildCommandConfigurations(appConf, getAmState().getInternalsSnapshot(), - containerId, roleName, roleGroup); - - cmd.setConfigurations(configurations); - - response.addStatusCommand(cmd); - } - - @VisibleForTesting - protected void addStatusDockerCommand(String roleName, - String roleGroup, - String containerId, - HeartBeatResponse response, - String scriptPath, - long timeout) - throws SliderException { - assert getAmState().isApplicationLive(); - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - - StatusCommand cmd = new StatusCommand(); - String clusterName = getClusterName(); - - cmd.setCommandType(AgentCommandType.STATUS_COMMAND); - cmd.setComponentName(roleName); - cmd.setServiceName(clusterName); - cmd.setClusterName(clusterName); - cmd.setRoleCommand(StatusCommand.STATUS_COMMAND); - - Map hostLevelParams = new TreeMap(); - hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME)); - hostLevelParams.put(CONTAINER_ID, containerId); - cmd.setHostLevelParams(hostLevelParams); - - cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false)); - - Map> configurations = buildCommandConfigurations( - appConf, getAmState().getInternalsSnapshot(), containerId, roleName, - roleGroup); - Map dockerConfig = new HashMap(); - String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, "statusCommand"); - if (statusCommand == null) { - if(isYarnDockerContainer(roleGroup)){ - //should complain the required field is null - cmd.setYarnDockerMode(true); - } else { - statusCommand = "docker top " - + containerId - + " | grep \"\"";// default value - } - } - dockerConfig.put("docker.status_command",statusCommand); - configurations.put("docker", dockerConfig); - cmd.setConfigurations(configurations); - log.debug("Docker- status {}", cmd); - response.addStatusCommand(cmd); - } - - @VisibleForTesting - protected void addGetConfigDockerCommand(String roleName, String roleGroup, - String containerId, HeartBeatResponse response) throws SliderException { - assert getAmState().isApplicationLive(); - - StatusCommand cmd = new StatusCommand(); - String clusterName = getClusterName(); - - cmd.setCommandType(AgentCommandType.STATUS_COMMAND); - cmd.setComponentName(roleName); - cmd.setServiceName(clusterName); - cmd.setClusterName(clusterName); - cmd.setRoleCommand(StatusCommand.GET_CONFIG_COMMAND); - Map hostLevelParams = new TreeMap(); - hostLevelParams.put(CONTAINER_ID, containerId); - cmd.setHostLevelParams(hostLevelParams); - - hostLevelParams.put(CONTAINER_ID, containerId); - - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - Map> configurations = buildCommandConfigurations( - appConf, getAmState().getInternalsSnapshot(), containerId, roleName, - roleGroup); - Map dockerConfig = new HashMap(); - String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, "statusCommand"); - if (statusCommand == null) { - if(isYarnDockerContainer(roleGroup)){ - //should complain the required field is null - cmd.setYarnDockerMode(true); - } else { - statusCommand = "docker top " - + containerId - + " | grep \"\"";// default value - } - } - dockerConfig.put("docker.status_command",statusCommand); - configurations.put("docker", dockerConfig); - - cmd.setConfigurations(configurations); - log.debug("Docker- getconfig command {}", cmd); - - response.addStatusCommand(cmd); - } - - private String getConfigFromMetaInfoWithAppConfigOverriding(String roleGroup, - String configName){ - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - String containerName = getApplicationComponent(roleGroup) - .getDockerContainers().get(0).getName(); - String composedConfigName = null; - String appConfigValue = null; - //if the configName is about port , mount, inputfile, then check differently - if (configName.equals("containerPort") || configName.equals("hostPort")){ - composedConfigName = containerName + ".ports." + configName; - } else - if (configName.equals("containerMount") - || configName.equals("hostMount")){ - composedConfigName = containerName + ".mounts." + configName; - } else - if (configName.equals("containerPath") - || configName.equals("fileLocalPath")) { - composedConfigName = containerName + ".inputFiles." + configName; - } else { - composedConfigName = containerName + "." + configName; - } - appConfigValue = appConf.getComponentOpt(roleGroup, composedConfigName, - null); - log.debug( - "Docker- value from appconfig component: {} configName: {} value: {}", - roleGroup, composedConfigName, appConfigValue); - if (appConfigValue == null) { - appConfigValue = getConfigFromMetaInfo(roleGroup, configName); - log.debug( - "Docker- value from metainfo component: {} configName: {} value: {}", - roleGroup, configName, appConfigValue); - - } - return appConfigValue; - } - - @VisibleForTesting - protected void addStartDockerCommand(String roleName, String roleGroup, - String containerId, HeartBeatResponse response, - ComponentCommand startCommand, long timeout, boolean isMarkedAutoRestart) - throws - SliderException { - assert getAmState().isApplicationLive(); - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot(); - - ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND); - prepareExecutionCommand(cmd); - String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME); - String hostName = getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME); - cmd.setHostname(hostName); - cmd.setClusterName(clusterName); - cmd.setRoleCommand(Command.START.toString()); - cmd.setServiceName(clusterName); - cmd.setComponentName(roleName); - cmd.setRole(roleName); - Map hostLevelParams = new TreeMap<>(); - hostLevelParams.put(CONTAINER_ID, containerId); - cmd.setHostLevelParams(hostLevelParams); - - Map roleParams = new TreeMap<>(); - cmd.setRoleParams(roleParams); - cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart)); - startCommand = new ComponentCommand(); - startCommand.setName("START"); - startCommand.setType("docker"); - startCommand.setExec("exec"); - cmd.setCommandParams(setCommandParameters(startCommand, timeout, true)); - - Map> configurations = buildCommandConfigurations( - appConf, getAmState().getInternalsSnapshot(), containerId, roleName, - roleGroup); - Map> componentConfigurations = buildComponentConfigurations(appConf); - cmd.setComponentConfigurations(componentConfigurations); - - Map dockerConfig = new HashMap(); - if (isYarnDockerContainer(roleGroup)) { - dockerConfig.put( - "docker.startCommand", - getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, - "start_command")); - cmd.setYarnDockerMode(true); - } else { - dockerConfig.put( - "docker.command_path", - getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, - "commandPath")); - - dockerConfig.put("docker.image_name", - getConfigFromMetaInfo(roleGroup, "image")); - // options should always have -d - String options = getConfigFromMetaInfoWithAppConfigOverriding( - roleGroup, "options"); - if(options != null && !options.isEmpty()){ - options = options + " -d"; - } else { - options = "-d"; - } - dockerConfig.put("docker.options", options); - // options should always have -d - dockerConfig.put( - "docker.containerPort", - getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, - "containerPort")); - dockerConfig - .put( - "docker.hostPort", - getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, - "hostPort")); - - dockerConfig.put( - "docker.mounting_directory", - getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, - "containerMount")); - dockerConfig - .put( - "docker.host_mounting_directory", - getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, - "hostMount")); - - dockerConfig.put("docker.additional_param", - getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, "additionalParam")); - - dockerConfig.put("docker.input_file.mount_path", getConfigFromMetaInfo( - roleGroup, "containerPath")); - } - - String lifetime = getConfigFromMetaInfoWithAppConfigOverriding( - roleGroup, "lifetime"); - dockerConfig.put("docker.lifetime", lifetime); - configurations.put("docker", dockerConfig); - String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding( - roleGroup, "statusCommand"); - if (statusCommand == null) { - if(isYarnDockerContainer(roleGroup)){ - //should complain the required field is null - } else { - statusCommand = "docker top " - + containerId + " | grep \"\""; - } - } - dockerConfig.put("docker.status_command",statusCommand); - - cmd.setConfigurations(configurations); - // configurations.get("global").put("exec_cmd", startCommand.getExec()); - cmd.addContainerDetails(roleGroup, getMetaInfo(roleGroup)); - - log.info("Docker- command: {}", cmd.toString()); - - response.addExecutionCommand(cmd); - } - - private String getConfigFromMetaInfo(String roleGroup, String configName) { - String result = null; - - List containers = getApplicationComponent( - roleGroup).getDockerContainers();// to support multi container per - // component later - log.debug("Docker- containers metainfo: {}", containers.toString()); - if (containers.size() > 0) { - DockerContainer container = containers.get(0); - switch (configName) { - case "start_command": - result = container.getStartCommand(); - break; - case "image": - result = container.getImage(); - break; - case "network": - if (container.getNetwork() == null || container.getNetwork().isEmpty()) { - result = "none"; - } else { - result = container.getNetwork(); - } - break; - case "statusCommand": - result = container.getStatusCommand(); - break; - case "commandPath": - result = container.getCommandPath(); - break; - case "options": - result = container.getOptions(); - break; - case "containerPort": - result = container.getPorts().size() > 0 ? container.getPorts().get(0) - .getContainerPort() : null;// to support - // multi port - // later - break; - case "hostPort": - result = container.getPorts().size() > 0 ? container.getPorts().get(0) - .getHostPort() : null;// to support multi - // port later - break; - case "containerMount": - result = container.getMounts().size() > 0 ? container.getMounts() - .get(0).getContainerMount() : null;// to support - // multi port - // later - break; - case "hostMount": - result = container.getMounts().size() > 0 ? container.getMounts() - .get(0).getHostMount() : null;// to support multi - // port later - break; - case "additionalParam": - result = container.getAdditionalParam();// to support multi port later - break; - case "runPriviledgedContainer": - if (container.getRunPrivilegedContainer() == null) { - result = "false"; - } else { - result = container.getRunPrivilegedContainer(); - } - break; - default: - break; - } - } - log.debug("Docker- component: {} configName: {} value: {}", roleGroup, configName, result); - return result; - } - - @VisibleForTesting - protected void addGetConfigCommand(String roleName, String roleGroup, - String containerId, HeartBeatResponse response) throws SliderException { - assert getAmState().isApplicationLive(); - - StatusCommand cmd = new StatusCommand(); - String clusterName = getClusterName(); - - cmd.setCommandType(AgentCommandType.STATUS_COMMAND); - cmd.setComponentName(roleName); - cmd.setServiceName(clusterName); - cmd.setClusterName(clusterName); - cmd.setRoleCommand(StatusCommand.GET_CONFIG_COMMAND); - Map hostLevelParams = new TreeMap(); - hostLevelParams.put(CONTAINER_ID, containerId); - cmd.setHostLevelParams(hostLevelParams); - - hostLevelParams.put(CONTAINER_ID, containerId); - - response.addStatusCommand(cmd); - } - - @VisibleForTesting - protected void addStartCommand(String roleName, String roleGroup, String containerId, - HeartBeatResponse response, - String scriptPath, ComponentCommand startCommand, - ComponentCommand stopCommand, - long timeout, boolean isMarkedAutoRestart) - throws - SliderException { - assert getAmState().isApplicationLive(); - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot(); - - ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND); - prepareExecutionCommand(cmd); - String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME); - String hostName = getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME); - cmd.setHostname(hostName); - cmd.setClusterName(clusterName); - cmd.setRoleCommand(Command.START.toString()); - cmd.setServiceName(clusterName); - cmd.setComponentName(roleName); - cmd.setRole(roleName); - Map hostLevelParams = new TreeMap<>(); - hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir())); - hostLevelParams.put(CONTAINER_ID, containerId); - cmd.setHostLevelParams(hostLevelParams); - - Map roleParams = new TreeMap<>(); - cmd.setRoleParams(roleParams); - cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart)); - - Map> configurations = - buildCommandConfigurations(appConf, internalsConf, containerId, - roleName, roleGroup); - cmd.setConfigurations(configurations); - Map> componentConfigurations = buildComponentConfigurations(appConf); - cmd.setComponentConfigurations(componentConfigurations); - - if (SliderUtils.isSet(scriptPath)) { - cmd.setCommandParams(commandParametersSet(scriptPath, timeout, true)); - } else { - if (startCommand == null) { - throw new SliderException("Expected START command not found for component " + roleName); - } - cmd.setCommandParams(commandParametersSet(startCommand, timeout, true)); - configurations.get("global").put("exec_cmd", startCommand.getExec()); - } - - response.addExecutionCommand(cmd); - - log.debug("command looks like: {}", cmd); - // With start command, the corresponding command for graceful stop needs to - // be sent. This will be used when a particular container is lost as per RM, - // but then the agent is still running and heart-beating to the Slider AM. - ExecutionCommand cmdStop = new ExecutionCommand( - AgentCommandType.EXECUTION_COMMAND); - cmdStop.setTaskId(taskId.get()); - cmdStop.setCommandId(cmdStop.getTaskId() + "-1"); - cmdStop.setHostname(hostName); - cmdStop.setClusterName(clusterName); - cmdStop.setRoleCommand(Command.STOP.toString()); - cmdStop.setServiceName(clusterName); - cmdStop.setComponentName(roleName); - cmdStop.setRole(roleName); - Map hostLevelParamsStop = new TreeMap(); - hostLevelParamsStop.put(JAVA_HOME, appConf.getGlobalOptions() - .getOption(JAVA_HOME, "")); - hostLevelParamsStop.put(CONTAINER_ID, containerId); - cmdStop.setHostLevelParams(hostLevelParamsStop); - - Map roleParamsStop = new TreeMap(); - cmdStop.setRoleParams(roleParamsStop); - cmdStop.getRoleParams().put("auto_restart", - Boolean.toString(isMarkedAutoRestart)); - - if (SliderUtils.isSet(scriptPath)) { - cmdStop.setCommandParams(commandParametersSet(scriptPath, timeout, true)); - } else { - if (stopCommand == null) { - stopCommand = ComponentCommand.getDefaultComponentCommand("STOP"); - } - cmd.setCommandParams(commandParametersSet(stopCommand, timeout, true)); - configurations.get("global").put("exec_cmd", startCommand.getExec()); - } - - - Map> configurationsStop = buildCommandConfigurations( - appConf, internalsConf, containerId, roleName, roleGroup); - cmdStop.setConfigurations(configurationsStop); - response.addExecutionCommand(cmdStop); - } - - @VisibleForTesting - protected void addUpgradeCommand(String roleName, String roleGroup, String containerId, - HeartBeatResponse response, String scriptPath, long timeout) - throws SliderException { - assert getAmState().isApplicationLive(); - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot(); - - ExecutionCommand cmd = new ExecutionCommand( - AgentCommandType.EXECUTION_COMMAND); - prepareExecutionCommand(cmd); - String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME); - String hostName = getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME); - cmd.setHostname(hostName); - cmd.setClusterName(clusterName); - cmd.setRoleCommand(Command.UPGRADE.toString()); - cmd.setServiceName(clusterName); - cmd.setComponentName(roleName); - cmd.setRole(roleName); - Map hostLevelParams = new TreeMap(); - hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions() - .getMandatoryOption(JAVA_HOME)); - hostLevelParams.put(CONTAINER_ID, containerId); - cmd.setHostLevelParams(hostLevelParams); - cmd.setCommandParams(commandParametersSet(scriptPath, timeout, true)); - - Map> configurations = buildCommandConfigurations( - appConf, internalsConf, containerId, roleName, roleGroup); - cmd.setConfigurations(configurations); - response.addExecutionCommand(cmd); - } - - protected void addStopCommand(String roleName, String roleGroup, String containerId, - HeartBeatResponse response, String scriptPath, long timeout, - boolean isInUpgradeMode) throws SliderException { - assert getAmState().isApplicationLive(); - ConfTreeOperations appConf = getAmState().getAppConfSnapshot(); - ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot(); - - ExecutionCommand cmdStop = new ExecutionCommand( - AgentCommandType.EXECUTION_COMMAND); - cmdStop.setTaskId(taskId.get()); - cmdStop.setCommandId(cmdStop.getTaskId() + "-1"); - String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME); - String hostName = getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME); - cmdStop.setHostname(hostName); - cmdStop.setClusterName(clusterName); - // Upgrade stop is differentiated by passing a transformed role command - - // UPGRADE_STOP - cmdStop.setRoleCommand(Command.transform(Command.STOP, isInUpgradeMode)); - cmdStop.setServiceName(clusterName); - cmdStop.setComponentName(roleName); - cmdStop.setRole(roleName); - Map hostLevelParamsStop = new TreeMap(); - hostLevelParamsStop.put(JAVA_HOME, appConf.getGlobalOptions() - .getMandatoryOption(JAVA_HOME)); - hostLevelParamsStop.put(CONTAINER_ID, containerId); - cmdStop.setHostLevelParams(hostLevelParamsStop); - cmdStop.setCommandParams(commandParametersSet(scriptPath, timeout, true)); - - Map> configurationsStop = buildCommandConfigurations( - appConf, internalsConf, containerId, roleName, roleGroup); - cmdStop.setConfigurations(configurationsStop); - response.addExecutionCommand(cmdStop); - } - - protected static String getJDKDir() { - File javaHome = new File(System.getProperty("java.home")).getParentFile(); - File jdkDirectory = null; - if (javaHome.getName().contains("jdk")) { - jdkDirectory = javaHome; - } - if (jdkDirectory != null) { - return jdkDirectory.getAbsolutePath(); - } else { - return ""; - } - } - - protected Map getAllocatedPorts() { - return getAllocatedPorts(SHARED_PORT_TAG); - } - - protected Map> getComponentInstanceData() { - return this.componentInstanceData; - } - - protected Map getAllocatedPorts(String containerId) { - if (!this.allocatedPorts.containsKey(containerId)) { - synchronized (this.allocatedPorts) { - if (!this.allocatedPorts.containsKey(containerId)) { - this.allocatedPorts.put(containerId, - new ConcurrentHashMap()); - } - } - } - return this.allocatedPorts.get(containerId); - } - - private Map> buildCommandConfigurations( - ConfTreeOperations appConf, ConfTreeOperations internalsConf, - String containerId, String roleName, String roleGroup) - throws SliderException { - - Map> configurations = new TreeMap<>(); - Map tokens = providerUtils.getStandardTokenMap(appConf, - internalsConf, roleName, roleGroup, getClusterName()); - tokens.put("${CONTAINER_ID}", containerId); - - Set configs = new HashSet(); - configs.addAll(getApplicationConfigurationTypes(roleGroup)); - configs.addAll(getSystemConfigurationsRequested(appConf)); - - for (String configType : configs) { - addNamedConfiguration(configType, appConf.getGlobalOptions().options, - configurations, tokens, containerId, roleName, - roleGroup); - if (appConf.getComponent(roleGroup) != null) { - addNamedConfiguration(configType, appConf.getComponent(roleGroup).options, - configurations, tokens, containerId, roleName, roleGroup); - } - } - - //do a final replacement of re-used configs - dereferenceAllConfigs(configurations); - - return configurations; - } - - @VisibleForTesting - protected void dereferenceAllConfigs(Map> configurations) { - providerUtils.dereferenceAllConfigs(configurations); - } - - @VisibleForTesting - protected Set getSystemConfigurationsRequested(ConfTreeOperations appConf) { - return providerUtils.getSystemConfigurationsRequested(appConf); - } - - @VisibleForTesting - protected List getApplicationConfigurationTypes(String roleGroup) { - List configList = new ArrayList(); - configList.add(GLOBAL_CONFIG_TAG); - - List configFiles = getMetaInfo(roleGroup).getApplication().getConfigFiles(); - for (ConfigFile configFile : configFiles) { - log.info("Expecting config type {}.", configFile.getDictionaryName()); - configList.add(configFile.getDictionaryName()); - } - for (Component component : getMetaInfo(roleGroup).getApplication().getComponents()) { - if (!component.getName().equals(roleGroup)) { - continue; - } - if (component.getDockerContainers() == null) { - continue; - } - for (DockerContainer container : component.getDockerContainers()) { - if (container.getConfigFiles() == null) { - continue; - } - for (ConfigFile configFile : container.getConfigFiles()) { - log.info("Expecting config type {}.", configFile.getDictionaryName()); - configList.add(configFile.getDictionaryName()); - } - } - } - - // remove duplicates. mostly worried about 'global' being listed - return new ArrayList(new HashSet(configList)); - } - - private void addNamedConfiguration(String configName, Map sourceConfig, - Map> configurations, - Map tokens, String containerId, - String roleName, String roleGroup) { - Map config = new HashMap(); - if (configName.equals(GLOBAL_CONFIG_TAG)) { - addDefaultGlobalConfig(config, containerId, roleName); - } - // add role hosts to tokens - addRoleRelatedTokens(tokens); - providerUtils.propagateSiteOptions(sourceConfig, config, configName, tokens); - - //apply any port updates - if (!this.getAllocatedPorts().isEmpty()) { - for (String key : config.keySet()) { - String value = config.get(key); - String lookupKey = configName + "." + key; - if (!value.contains(PER_CONTAINER_TAG)) { - // If the config property is shared then pass on the already allocated value - // from any container - if (this.getAllocatedPorts().containsKey(lookupKey)) { - config.put(key, getAllocatedPorts().get(lookupKey)); - } - } else { - if (this.getAllocatedPorts(containerId).containsKey(lookupKey)) { - config.put(key, getAllocatedPorts(containerId).get(lookupKey)); - } - } - } - } - - //apply defaults only if the key is not present and value is not empty - if (getDefaultConfigs(roleGroup).containsKey(configName)) { - log.info("Adding default configs for type {}.", configName); - for (PropertyInfo defaultConfigProp : getDefaultConfigs(roleGroup).get(configName).getPropertyInfos()) { - if (!config.containsKey(defaultConfigProp.getName())) { - if (!defaultConfigProp.getName().isEmpty() && - defaultConfigProp.getValue() != null && - !defaultConfigProp.getValue().isEmpty()) { - config.put(defaultConfigProp.getName(), defaultConfigProp.getValue()); - } - } - } - } - - configurations.put(configName, config); - } - - @VisibleForTesting - protected void addRoleRelatedTokens(Map tokens) { - providerUtils.addRoleRelatedTokens(tokens, getAmState()); - } - - private void addDefaultGlobalConfig(Map config, String containerId, String roleName) { - config.put("app_log_dir", "${AGENT_LOG_ROOT}"); - config.put("app_pid_dir", "${AGENT_WORK_ROOT}/app/run"); - config.put("app_install_dir", "${AGENT_WORK_ROOT}/app/install"); - config.put("app_conf_dir", "${AGENT_WORK_ROOT}/" + APP_CONF_DIR); - config.put("app_input_conf_dir", "${AGENT_WORK_ROOT}/" + PROPAGATED_CONF_DIR_NAME); - config.put("app_container_id", containerId); - config.put("app_container_tag", tags.getTag(roleName, containerId)); - - // add optional parameters only if they are not already provided - if (!config.containsKey("pid_file")) { - config.put("pid_file", "${AGENT_WORK_ROOT}/app/run/component.pid"); - } - if (!config.containsKey("app_root")) { - config.put("app_root", "${AGENT_WORK_ROOT}/app/install"); - } - } - - private void buildRoleHostDetails(Map details) { - for (Map.Entry> entry : - getRoleClusterNodeMapping().entrySet()) { - details.put(entry.getKey() + " Host(s)/Container(s)", - new MonitorDetail(providerUtils.getHostsList( - entry.getValue().values(), false).toString(), false)); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java deleted file mode 100644 index 281895a32a4..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent; - -import org.apache.slider.providers.ProviderRole; - -import java.util.ArrayList; -import java.util.List; - -public class AgentRoles { - - /** - * List of roles Agent provider does not have any roles by default. All roles are read from the application - * specification. - */ - protected static final List ROLES = - new ArrayList(); - - public static List getRoles() { - return ROLES; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java deleted file mode 100644 index 23e05a38bdd..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent; - -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.providers.agent.application.metadata.AbstractMetainfoParser; -import org.apache.slider.providers.agent.application.metadata.AddonPackageMetainfoParser; -import org.apache.slider.providers.agent.application.metadata.DefaultConfig; -import org.apache.slider.providers.agent.application.metadata.DefaultConfigParser; -import org.apache.slider.providers.agent.application.metadata.Metainfo; -import org.apache.slider.providers.agent.application.metadata.MetainfoParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; - -import static org.apache.slider.api.RoleKeys.ROLE_PREFIX; - -/** - * - */ -public class AgentUtils { - private static final Logger log = LoggerFactory.getLogger(AgentUtils.class); - - public static Metainfo getApplicationMetainfoFromSummaryFile( - SliderFileSystem fileSystem, String metainfoPath, boolean metainfoForAddon) { - FileSystem fs = fileSystem.getFileSystem(); - Path appPathXML = new Path(metainfoPath + ".metainfo.xml"); - Path appPathJson = new Path(metainfoPath + ".metainfo.json"); - Path appPathUsed = null; - try { - FSDataInputStream appStream = null; - if (fs.exists(appPathXML)) { - appPathUsed = appPathXML; - appStream = fs.open(appPathXML); - return parseMetainfo(appStream, metainfoForAddon, "xml"); - } else if (fs.exists(appPathJson)) { - appPathUsed = appPathJson; - appStream = fs.open(appPathJson); - return parseMetainfo(appStream, metainfoForAddon, "json"); - } - } catch (IOException e) { - log.info("Failed to get metainfo from summary file {} - {}", appPathUsed, - e.getMessage()); - log.debug("Failed to get metainfo", e); - } - return null; - } - - public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, - String metainfoPath, boolean metainfoForAddon) throws IOException, - BadConfigException { - log.info("Reading metainfo at {}", metainfoPath); - Metainfo metainfo = getApplicationMetainfoFromSummaryFile(fileSystem, - metainfoPath, metainfoForAddon); - if (metainfo != null) { - log.info("Got metainfo from summary file"); - return metainfo; - } - - FileSystem fs = fileSystem.getFileSystem(); - Path appPath = new Path(metainfoPath); - - InputStream metainfoJsonStream = SliderUtils.getApplicationResourceInputStream( - fs, appPath, "metainfo.json"); - if (metainfoJsonStream == null) { - InputStream metainfoXMLStream = SliderUtils.getApplicationResourceInputStream( - fs, appPath, "metainfo.xml"); - if (metainfoXMLStream != null) { - metainfo = parseMetainfo(metainfoXMLStream, metainfoForAddon, "xml"); - } - } else { - metainfo = parseMetainfo(metainfoJsonStream, metainfoForAddon, "json"); - } - - if (metainfo == null) { - log.error("metainfo is unavailable at {}.", metainfoPath); - throw new FileNotFoundException("metainfo.xml/json is required in app package. " + - appPath); - } - return metainfo; - } - - private static Metainfo parseMetainfo(InputStream stream, - boolean metainfoForAddon, String type) throws IOException { - AbstractMetainfoParser metainfoParser = null; - if (metainfoForAddon) { - metainfoParser = new AddonPackageMetainfoParser(); - } else { - metainfoParser = new MetainfoParser(); - } - if (type.equals("xml")) { - return metainfoParser.fromXmlStream(stream); - } else if (type.equals("json")) { - return metainfoParser.fromJsonStream(stream); - } - return null; - } - - static DefaultConfig getDefaultConfig(SliderFileSystem fileSystem, - String appDef, String configFileName) - throws IOException { - // this is the path inside the zip file - String fileToRead = "configuration/" + configFileName; - log.info("Reading default config file {} at {}", fileToRead, appDef); - InputStream configStream = SliderUtils.getApplicationResourceInputStream( - fileSystem.getFileSystem(), new Path(appDef), fileToRead); - if (configStream == null) { - log.error("{} is unavailable at {}.", fileToRead, appDef); - throw new IOException("Expected config file " + fileToRead + " is not available."); - } - - return new DefaultConfigParser().parse(configStream); - } - - static String getMetainfoComponentName(String roleGroup, - ConfTreeOperations appConf) throws BadConfigException { - String prefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, null); - if (prefix == null) { - return roleGroup; - } - if (!roleGroup.startsWith(prefix)) { - throw new BadConfigException("Component " + roleGroup + " doesn't start" + - " with prefix " + prefix); - } - return roleGroup.substring(prefix.length()); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/Command.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/Command.java deleted file mode 100644 index 647cb8624f5..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/Command.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent; - -/** The states a component instance can be. */ -public enum Command { - NOP, // do nothing - INSTALL, // Install the component - INSTALL_ADDON, // Install add on packages if any - START, // Start the component - STOP, // Stop the component - UPGRADE, // The component will undergo upgrade - TERMINATE; // Send terminate signal to agent - - public static Command getCommand(String commandVal) { - if (commandVal.equals(Command.START.toString())) { - return Command.START; - } - if (commandVal.equals(Command.INSTALL.toString())) { - return Command.INSTALL; - } - if (commandVal.equals(Command.STOP.toString())) { - return Command.STOP; - } - if (commandVal.equals(Command.UPGRADE.toString())) { - return Command.UPGRADE; - } - if (commandVal.equals(Command.TERMINATE.toString())) { - return Command.TERMINATE; - } - - return Command.NOP; - } - - public static String transform(Command command, boolean isUpgrade) { - switch (command) { - case STOP: - return isUpgrade ? "UPGRADE_STOP" : command.name(); - default: - return command.name(); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java deleted file mode 100644 index 35d9116ed68..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent; - -/** Command results. */ -public enum CommandResult { - IN_PROGRESS, // Command is in progress - COMPLETED, // Command has successfully completed - FAILED; // Command has failed - - public static CommandResult getCommandResult(String commandResVal) { - if (commandResVal.equals(CommandResult.COMPLETED.toString())) { - return CommandResult.COMPLETED; - } - if (commandResVal.equals(CommandResult.FAILED.toString())) { - return CommandResult.FAILED; - } - if (commandResVal.equals(CommandResult.IN_PROGRESS.toString())) { - return CommandResult.IN_PROGRESS; - } - - throw new IllegalArgumentException("Unrecognized value " + commandResVal); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java deleted file mode 100644 index 4abac7a18ba..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent; - -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.providers.agent.application.metadata.CommandOrder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; - -import static org.apache.slider.api.RoleKeys.ROLE_PREFIX; - -/** - * Stores the command dependency order for all components in a service. - * SUPERVISOR-START NIMBUS-STARTED Means, SUPERVISOR START - * requires NIMBUS to be STARTED - */ -public class ComponentCommandOrder { - public static final Logger log = - LoggerFactory.getLogger(ComponentCommandOrder.class); - private static char SPLIT_CHAR = '-'; - Map>> dependencies = - new HashMap>>(); - Map> prefixRoleMap = new HashMap<>(); - Map rolePrefixMap = new HashMap<>(); - - public ComponentCommandOrder() {} - - public ComponentCommandOrder(List commandOrders, - ConfTreeOperations resources) { - mergeCommandOrders(commandOrders, resources); - } - - void mergeCommandOrders(List commandOrders, - ConfTreeOperations resources) { - for (String component : resources.getComponentNames()) { - String prefix = SliderUtils.trimPrefix( - resources.getComponentOpt(component, ROLE_PREFIX, null)); - if (prefix != null) { - rolePrefixMap.put(component, prefix); - if (!prefixRoleMap.containsKey(prefix)) { - prefixRoleMap.put(prefix, new HashSet()); - } - prefixRoleMap.get(prefix).add(component); - } - } - if (commandOrders != null && commandOrders.size() > 0) { - for (CommandOrder commandOrder : commandOrders) { - ComponentCommand componentCmd = getComponentCommand( - commandOrder.getCommand(), resources); - String requires = commandOrder.getRequires(); - List requiredStates = parseRequiredStates(requires, - resources); - if (requiredStates.size() > 0) { - Map> compDep = dependencies.get(componentCmd.command); - if (compDep == null) { - compDep = new HashMap<>(); - dependencies.put(componentCmd.command, compDep); - } - - List requirements = compDep.get(componentCmd.componentName); - if (requirements == null) { - requirements = new ArrayList<>(); - compDep.put(componentCmd.componentName, requirements); - } - - requirements.addAll(requiredStates); - } - } - } - } - - private List parseRequiredStates(String requires, - ConfTreeOperations resources) { - if (requires == null || requires.length() < 2) { - throw new IllegalArgumentException("Input cannot be null and must contain component and state."); - } - - String[] componentStates = requires.split(","); - List retList = new ArrayList(); - for (String componentStateStr : componentStates) { - retList.add(getComponentState(componentStateStr, resources)); - } - - return retList; - } - - private ComponentCommand getComponentCommand(String compCmdStr, - ConfTreeOperations resources) { - if (compCmdStr == null || compCmdStr.trim().length() < 2) { - throw new IllegalArgumentException("Input cannot be null and must contain component and command."); - } - - compCmdStr = compCmdStr.trim(); - int splitIndex = compCmdStr.lastIndexOf(SPLIT_CHAR); - if (splitIndex == -1 || splitIndex == 0 || splitIndex == compCmdStr.length() - 1) { - throw new IllegalArgumentException("Input does not appear to be well-formed."); - } - String compStr = compCmdStr.substring(0, splitIndex); - String cmdStr = compCmdStr.substring(splitIndex + 1); - - if (resources.getComponent(compStr) == null && !prefixRoleMap.containsKey(compStr)) { - throw new IllegalArgumentException("Component " + compStr + " specified" + - " in command order does not exist"); - } - - Command cmd = Command.valueOf(cmdStr); - - if (cmd != Command.START) { - throw new IllegalArgumentException("Dependency order can only be specified for START."); - } - return new ComponentCommand(compStr, cmd); - } - - private ComponentState getComponentState(String compStStr, - ConfTreeOperations resources) { - if (compStStr == null || compStStr.trim().length() < 2) { - throw new IllegalArgumentException("Input cannot be null."); - } - - compStStr = compStStr.trim(); - int splitIndex = compStStr.lastIndexOf(SPLIT_CHAR); - if (splitIndex == -1 || splitIndex == 0 || splitIndex == compStStr.length() - 1) { - throw new IllegalArgumentException("Input does not appear to be well-formed."); - } - String compStr = compStStr.substring(0, splitIndex); - String stateStr = compStStr.substring(splitIndex + 1); - - if (resources.getComponent(compStr) == null && !prefixRoleMap.containsKey(compStr)) { - throw new IllegalArgumentException("Component " + compStr + " specified" + - " in command order does not exist"); - } - - State state = State.valueOf(stateStr); - if (state != State.STARTED && state != State.INSTALLED) { - throw new IllegalArgumentException("Dependency order can only be specified against STARTED/INSTALLED."); - } - return new ComponentState(compStr, state); - } - - // dependency is still on component level, but not package level - // so use component name to check dependency, not component-package - public boolean canExecute(String component, Command command, Collection currentStates) { - if (!dependencies.containsKey(command)) { - return true; - } - List required = new ArrayList<>(); - if (dependencies.get(command).containsKey(component)) { - required.addAll(dependencies.get(command).get(component)); - } - String prefix = rolePrefixMap.get(component); - if (prefix != null && dependencies.get(command).containsKey(prefix)) { - required.addAll(dependencies.get(command).get(prefix)); - } - - for (ComponentState stateToMatch : required) { - for (ComponentInstanceState currState : currentStates) { - log.debug("Checking schedule {} {} against dependency {} is {}", - component, command, currState.getComponentName(), currState.getState()); - if (currState.getComponentName().equals(stateToMatch.componentName) || - (prefixRoleMap.containsKey(stateToMatch.componentName) && - prefixRoleMap.get(stateToMatch.componentName).contains(currState.getComponentName()))) { - if (currState.getState() != stateToMatch.state) { - if (stateToMatch.state == State.STARTED) { - log.info("Cannot schedule {} {} as dependency {} is {}", - component, command, currState.getComponentName(), currState.getState()); - return false; - } else { - //state is INSTALLED - if (currState.getState() != State.STARTING && currState.getState() != State.STARTED) { - log.info("Cannot schedule {} {} as dependency {} is {}", - component, command, currState.getComponentName(), currState.getState()); - return false; - } - } - } - } - } - } - return true; - } - - static class ComponentState { - public String componentName; - public State state; - - public ComponentState(String componentName, State state) { - this.componentName = componentName; - this.state = state; - } - } - - static class ComponentCommand { - public String componentName; - public Command command; - - public ComponentCommand(String componentName, Command command) { - this.componentName = componentName; - this.command = command; - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java deleted file mode 100644 index 6ee0ebb855f..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java +++ /dev/null @@ -1,340 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent; - -import java.util.Map; -import java.util.TreeMap; - -import com.google.common.annotations.VisibleForTesting; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.slider.providers.agent.application.metadata.Component; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** This class implements a simple state machine for component instances. */ -public class ComponentInstanceState { - public static final Logger log = - LoggerFactory.getLogger(ComponentInstanceState.class); - private static int MAX_FAILURE_TOLERATED = 3; - private static String INVALID_TRANSITION_ERROR = - "Result %s for command %s is not expected for component %s in state %s."; - - private final String componentName; - private final ContainerId containerId; - private final String containerIdAsString; - private final String applicationId; - private State state = State.INIT; - private State targetState = State.STARTED; - private int failuresSeen = 0; - private Boolean configReported = false; - private long lastHeartbeat = 0; - private String ip; - private String hostname; - private ContainerState containerState; - - private Map pkgStatuses; - private String nextPkgToInstall; - - private boolean stopInitiated; - - public ComponentInstanceState(String componentName, - ContainerId containerId, - String applicationId) { - this(componentName, containerId, applicationId, - new TreeMap()); - } - - public ComponentInstanceState(String componentName, - ContainerId containerId, - String applicationId, Map pkgStatuses) { - this.componentName = componentName; - this.containerId = containerId; - this.containerIdAsString = containerId.toString(); - this.applicationId = applicationId; - this.containerState = ContainerState.INIT; - this.lastHeartbeat = System.currentTimeMillis(); - this.pkgStatuses = pkgStatuses; - } - - public String getComponentName() { - return componentName; - } - - public Boolean getConfigReported() { - return configReported; - } - - public void setConfigReported(Boolean configReported) { - this.configReported = configReported; - } - - public ContainerState getContainerState() { - return containerState; - } - - public void setContainerState(ContainerState containerState) { - this.containerState = containerState; - } - - public long getLastHeartbeat() { - return lastHeartbeat; - } - - /** - * Update the heartbeat, and change container state - * to mark as healthy if appropriate - * @param heartbeatTime last time the heartbeat was seen - * @return the current container state - */ - public ContainerState heartbeat(long heartbeatTime) { - this.lastHeartbeat = heartbeatTime; - if(containerState == ContainerState.UNHEALTHY || - containerState == ContainerState.INIT) { - containerState = ContainerState.HEALTHY; - } - return containerState; - } - - - public ContainerId getContainerId() { - return containerId; - } - - public void commandIssued(Command command) { - commandIssued(command, false); - } - - public void commandIssued(Command command, boolean isInUpgradeMode) { - Command expected = getNextCommand(isInUpgradeMode); - if (expected != command) { - throw new IllegalArgumentException("Command " + command + " is not allowed in state " + state); - } - if (expected == Command.INSTALL_ADDON) { - // for add on packages, the pkg must be nextPkgToInstall - State currentState = pkgStatuses.get(nextPkgToInstall); - log.debug("Command issued: component: {} is in {}", componentName, - currentState); - State nextState = currentState.getNextState(command); - pkgStatuses.put(nextPkgToInstall, nextState); - log.debug("Command issued: component: {} is now in {}", componentName, - nextState); - } else { - // for master package - state = state.getNextState(command); - } - } - - public void applyCommandResult(CommandResult result, Command command, - String pkg) { - // if the heartbeat is for a package - // update that package's state in the component status - // and don't bother with the master pkg - if (StringUtils.isNotEmpty(pkg) - && !Component.MASTER_PACKAGE_NAME.equals(pkg)) { - log.debug("This result is for component: {} pkg: {}", componentName, pkg); - State previousPkgState = pkgStatuses.get(pkg); - log.debug("Currently component: {} pkg: {} is in state: {}", - componentName, pkg, previousPkgState.toString()); - State nextPkgState = previousPkgState.getNextState(result); - pkgStatuses.put(pkg, nextPkgState); - log.debug("Component: {} pkg: {} next state: {}", componentName, pkg, - nextPkgState); - } else { - log.debug("This result is for component: {} master package", - componentName); - applyCommandResult(result, command); - } - } - - public void applyCommandResult(CommandResult result, Command command) { - if (!this.state.couldHaveIssued(command)) { - throw new IllegalStateException("Invalid command " + command + " for state " + this.state); - } - - try { - if (result == CommandResult.FAILED) { - failuresSeen++; - } else if (result == CommandResult.COMPLETED) { - failuresSeen = 0; - } - state = state.getNextState(result); - } catch (IllegalArgumentException e) { - String message = String.format(INVALID_TRANSITION_ERROR, - result.toString(), - command.toString(), - componentName, - state.toString()); - log.warn(message); - throw new IllegalStateException(message); - } - } - - public boolean hasPendingCommand() { - if (state.canIssueCommands() && - state != targetState && - failuresSeen < MAX_FAILURE_TOLERATED) { - return true; - } - - return false; - } - - public Command getNextCommand() { - return getNextCommand(false); - } - - public Command getNextCommand(boolean isInUpgradeMode) { - if (!hasPendingCommand()) { - nextPkgToInstall = null; - return Command.NOP; - } - - log.debug("In getNextCommand, checking for component: {} ", componentName); - // if the master pkg is just installed, check if any add on pkg need to be - // installed - nextPkgToInstall = null; - if (state == State.INSTALLED) { - for (Map.Entry pkgStatus : pkgStatuses.entrySet()) { - String pkg = pkgStatus.getKey(); - State pkgState = pkgStatus.getValue(); - log.debug("In getNextCommand, pkg: {} is in {}", pkg, pkgState); - if (pkgState == State.INSTALLING) { - // first check if any pkg is install in progress, if so, wait - // so we don't need to do anything, just return NOP - log.debug("In getNextCommand, pkg: {} we are issuing NOP", pkg); - nextPkgToInstall = pkg; - return Command.NOP; - } else if (pkgState == State.INIT) { - // temporarily storing pkg here - // in case no pkg in 'installing' state - // will return the package to install - nextPkgToInstall = pkg; - } - } - // when we reach here, no pkg is in 'installing' state - if (nextPkgToInstall != null) { - // nextPkgToInstall != null means some pkg is in INIT state - // issue 'install' to the pkg we have stored in nextPkgToInstall - log.debug("In getNextCommand, pkg: {} we are issuing install addon", - nextPkgToInstall); - return Command.INSTALL_ADDON; - } - } - return this.state.getSupportedCommand(isInUpgradeMode, stopInitiated); - } - - public State getState() { - return state; - } - - @VisibleForTesting - protected void setState(State state) { - this.state = state; - } - - public State getTargetState() { - return targetState; - } - - public void setTargetState(State targetState) { - this.targetState = targetState; - } - - public String getNextPkgToInstall() { - return nextPkgToInstall; - } - - public boolean isStopInitiated() { - return stopInitiated; - } - - public void setStopInitiated(boolean stopInitiated) { - this.stopInitiated = stopInitiated; - } - - @Override - public int hashCode() { - int hashCode = 1; - - hashCode = hashCode ^ (componentName != null ? componentName.hashCode() : 0); - hashCode = hashCode ^ (containerIdAsString != null ? containerIdAsString.hashCode() : 0); - hashCode = hashCode ^ (applicationId != null ? applicationId.hashCode() : 0); - return hashCode; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - - if (o == null || getClass() != o.getClass()) return false; - - ComponentInstanceState that = (ComponentInstanceState) o; - - if (this.componentName != null ? - !this.componentName.equals(that.componentName) : this.componentName != null) { - return false; - } - - if (this.containerIdAsString != null ? - !this.containerIdAsString.equals(that.containerIdAsString) : this.containerIdAsString != null) { - return false; - } - - if (this.applicationId != null ? - !this.applicationId.equals(that.applicationId) : this.applicationId != null) { - return false; - } - - return true; - } - - @Override - public String toString() { - final StringBuilder sb = - new StringBuilder("ComponentInstanceState{"); - sb.append("containerIdAsString='").append(containerIdAsString).append('\''); - sb.append(", state=").append(state); - sb.append(", failuresSeen=").append(failuresSeen); - sb.append(", lastHeartbeat=").append(lastHeartbeat); - sb.append(", containerState=").append(containerState); - sb.append(", componentName='").append(componentName).append('\''); - sb.append(", ip=").append(ip); - sb.append(", hostname='").append(hostname).append('\''); - sb.append('}'); - return sb.toString(); - } - - public String getIp() { - return ip; - } - - public void setIp(String ip) { - this.ip = ip; - } - - public String getHostname() { - return hostname; - } - - public void setHostname(String hostname) { - this.hostname = hostname; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentTagProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentTagProvider.java deleted file mode 100644 index 68f63fa2c20..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentTagProvider.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent; - -import org.apache.slider.common.tools.SliderUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.ConcurrentHashMap; - -/** A simple tag provider that attempts to associate tags from 1-N to all container of a component */ -public class ComponentTagProvider { - private static final Logger log = LoggerFactory.getLogger(ComponentTagProvider.class); - private static String FREE = "free"; - private final ConcurrentHashMap> allTags; - - public ComponentTagProvider() { - allTags = new ConcurrentHashMap>(); - } - - /** - * Record an assigned tag to a container - * - * @param component - * @param containerId - * @param tag - */ - public void recordAssignedTag(String component, String containerId, String tag) { - if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) { - Integer key = null; - try { - key = Integer.valueOf(tag); - } catch (NumberFormatException nfe) { - //ignore - } - if (key != null && key > 0) { - ConcurrentHashMap compTags = getComponentSpecificTags(component); - synchronized (compTags) { - for (int index = 1; index <= key.intValue(); index++) { - String tempKey = new Integer(index).toString(); - if (!compTags.containsKey(tempKey)) { - compTags.put(tempKey, FREE); - } - } - compTags.put(key.toString(), containerId); - } - } - } - } - - /** - * Get a tag for container - * - * @param component - * @param containerId - * - * @return - */ - public String getTag(String component, String containerId) { - if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) { - ConcurrentHashMap compTags = getComponentSpecificTags(component); - synchronized (compTags) { - for (String key : compTags.keySet()) { - if (compTags.get(key).equals(containerId)) { - return key; - } - } - for (String key : compTags.keySet()) { - if (compTags.get(key).equals(FREE)) { - compTags.put(key, containerId); - return key; - } - } - String newKey = new Integer(compTags.size() + 1).toString(); - compTags.put(newKey, containerId); - return newKey; - } - } - return ""; - } - - /** - * Release a tag associated with a container - * - * @param component - * @param containerId - */ - public void releaseTag(String component, String containerId) { - if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) { - ConcurrentHashMap compTags = allTags.get(component); - if (compTags != null) { - synchronized (compTags) { - for (String key : compTags.keySet()) { - if (compTags.get(key).equals(containerId)) { - compTags.put(key, FREE); - } - } - } - } - } - } - - private ConcurrentHashMap getComponentSpecificTags(String component) { - if (!allTags.containsKey(component)) { - synchronized (allTags) { - if (!allTags.containsKey(component)) { - allTags.put(component, new ConcurrentHashMap()); - } - } - } - return allTags.get(component); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java deleted file mode 100644 index 0394ba2b070..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent; - -/** The states a component instance can be. */ -public enum ContainerState { - INIT, // Container is not net activated - HEALTHY, // Agent is heartbeating - UNHEALTHY, // Container is unhealthy - no heartbeat for some interval - HEARTBEAT_LOST; // Container is lost - request a new instance - - /** - * Indicates whether or not it is a valid state to produce a command. - * - * @return true if command can be issued for this state. - */ - public boolean canIssueCommands() { - switch (this) { - case HEALTHY: - return true; - default: - return false; - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java deleted file mode 100644 index 4293916ca89..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent; - -import com.google.common.annotations.VisibleForTesting; - -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; - -/** Monitors the container state and heartbeats. */ -public class HeartbeatMonitor implements Runnable { - protected static final Logger log = - LoggerFactory.getLogger(HeartbeatMonitor.class); - private final int threadWakeupInterval; //1 minute - private final AgentProviderService provider; - private volatile boolean shouldRun = true; - private Thread monitorThread = null; - - public HeartbeatMonitor(AgentProviderService provider, int threadWakeupInterval) { - this.provider = provider; - this.threadWakeupInterval = threadWakeupInterval; - } - - public void shutdown() { - shouldRun = false; - } - - public void start() { - log.info("Starting heartbeat monitor with interval {}", threadWakeupInterval); - monitorThread = new Thread(this); - monitorThread.start(); - } - - void join(long millis) throws InterruptedException { - if (isAlive()) { - monitorThread.join(millis); - } - } - - public boolean isAlive() { - return monitorThread != null && monitorThread.isAlive(); - } - - @Override - public void run() { - while (shouldRun) { - try { - log.debug("Putting monitor to sleep for " + threadWakeupInterval + " " + - "milliseconds"); - Thread.sleep(threadWakeupInterval); - doWork(System.currentTimeMillis()); - } catch (InterruptedException ex) { - log.warn("Scheduler thread is interrupted going to stop", ex); - shouldRun = false; - } catch (Exception ex) { - log.warn("Exception received", ex); - } catch (Throwable t) { - log.warn("ERROR", t); - } - } - } - - /** - * Every interval the current state of the container are checked. If the state is INIT or HEALTHY and no HB are - * received in last check interval they are marked as UNHEALTHY. INIT is when the agent is started but it did not - * communicate at all. HEALTHY being the AM has received heartbeats. After an interval as UNHEALTHY the container is - * declared unavailable - * @param now current time in milliseconds ... tests can set this explicitly - */ - @VisibleForTesting - public void doWork(long now) { - Map componentStatuses = provider.getComponentStatuses(); - if (componentStatuses != null) { - for (String containerLabel : componentStatuses.keySet()) { - ComponentInstanceState componentInstanceState = componentStatuses.get(containerLabel); - long timeSinceLastHeartbeat = now - componentInstanceState.getLastHeartbeat(); - - if (timeSinceLastHeartbeat > threadWakeupInterval) { - switch (componentInstanceState.getContainerState()) { - case INIT: - case HEALTHY: - componentInstanceState.setContainerState(ContainerState.UNHEALTHY); - log.warn( - "Component {} marked UNHEALTHY. Last heartbeat received at {} approx. {} ms. back.", - componentInstanceState, - componentInstanceState.getLastHeartbeat(), - timeSinceLastHeartbeat); - break; - case UNHEALTHY: - if (timeSinceLastHeartbeat > threadWakeupInterval * 2) { - componentInstanceState.setContainerState( - ContainerState.HEARTBEAT_LOST); - log.warn( - "Component {} marked HEARTBEAT_LOST. Last heartbeat received at {} approx. {} ms. back.", - componentInstanceState, componentInstanceState.getLastHeartbeat(), - timeSinceLastHeartbeat); - ContainerId containerId = - componentInstanceState.getContainerId(); - provider.lostContainer(containerLabel, containerId); - } - break; - case HEARTBEAT_LOST: - // unexpected case - log.warn("Heartbeat from lost component: {}", componentInstanceState); - break; - } - - } - } - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/State.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/State.java deleted file mode 100644 index 5603f8d50af..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/State.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent; - -/** The states a component instance can be. */ -public enum State { - INIT, // Not installed - INSTALLING, // Being installed - INSTALLED, // Installed (or stopped) - STARTING, // Starting - STARTED, // Started - INSTALL_FAILED, // Install failed, start failure in INSTALLED - UPGRADING, // Undergoing upgrade, perform necessary pre-upgrade steps - UPGRADED, // Pre-upgrade steps completed - STOPPING, // Stop has been issued - STOPPED, // Agent has stopped - TERMINATING; // Terminate signal to ask the agent to kill itself - // No need for state TERMINATED (as the agent is dead by then) - - /** - * Indicates whether or not it is a valid state to produce a command. - * - * @return true if command can be issued for this state. - */ - public boolean canIssueCommands() { - switch (this) { - case INSTALLING: - case STARTING: - case UPGRADING: - case STOPPING: - case TERMINATING: - return false; - default: - return true; - } - } - - /** - * Returns valid command in this state. - * - * @return command allowed in this state. - */ - public Command getSupportedCommand() { - return getSupportedCommand(false); - } - - public Command getSupportedCommand(boolean isInUpgradeMode) { - return getSupportedCommand(isInUpgradeMode, false); - } - - public Command getSupportedCommand(boolean isInUpgradeMode, - boolean stopInitiated) { - switch (this) { - case INIT: - case INSTALL_FAILED: - return Command.INSTALL; - case INSTALLED: - return Command.START; - case STARTED: - return isInUpgradeMode ? Command.UPGRADE : (stopInitiated) ? Command.STOP - : Command.NOP; - case UPGRADED: - return Command.STOP; - case STOPPED: - return Command.TERMINATE; - default: - return Command.NOP; - } - } - - /** - * Returns next state based on the command result. - * - * @return next state. - */ - public State getNextState(CommandResult result) throws IllegalArgumentException { - switch (result) { - case IN_PROGRESS: - if (this == State.INSTALLING || this == State.STARTING - || this == State.UPGRADING || this == State.STOPPING - || this == State.TERMINATING) { - return this; - } else { - throw new IllegalArgumentException(result + " is not valid for " + this); - } - case COMPLETED: - if (this == State.INSTALLING) { - return State.INSTALLED; - } else if (this == State.STARTING) { - return State.STARTED; - } else if (this == State.UPGRADING) { - return State.UPGRADED; - } else if (this == State.STOPPING) { - return State.STOPPED; - } else { - throw new IllegalArgumentException(result + " is not valid for " + this); - } - case FAILED: - if (this == State.INSTALLING) { - return State.INSTALL_FAILED; - } else if (this == State.STARTING) { - return State.INSTALLED; - } else if (this == State.UPGRADING) { - // if pre-upgrade failed, force stop now, so mark it upgraded - // what other options can be exposed to app owner? - return State.UPGRADED; - } else if (this == State.STOPPING) { - // if stop fails, force mark it stopped (and let container terminate) - return State.STOPPED; - } else if (this == State.STOPPED) { - // if in stopped state, force mark it as terminating - return State.TERMINATING; - } else { - throw new IllegalArgumentException(result + " is not valid for " + this); - } - default: - throw new IllegalArgumentException("Bad command result " + result); - } - } - - /** - * Returns next state based on the command. - * - * @return next state. - */ - public State getNextState(Command command) throws IllegalArgumentException { - switch (command) { - case INSTALL: - if (this == State.INIT || this == State.INSTALL_FAILED) { - return State.INSTALLING; - } else { - throw new IllegalArgumentException(command + " is not valid for " + this); - } - case INSTALL_ADDON: - if (this == State.INIT || this == State.INSTALL_FAILED) { - return State.INSTALLING; - } else { - throw new IllegalArgumentException(command + " is not valid for " + this); - } - case START: - if (this == State.INSTALLED) { - return State.STARTING; - } else { - throw new IllegalArgumentException(command + " is not valid for " + this); - } - case UPGRADE: - if (this == State.STARTED) { - return State.UPGRADING; - } else { - throw new IllegalArgumentException(command + " is not valid for " + this); - } - case STOP: - if (this == State.STARTED || this == State.UPGRADED) { - return State.STOPPING; - } else { - throw new IllegalArgumentException(command + " is not valid for " + this); - } - case TERMINATE: - if (this == State.STOPPED) { - return State.TERMINATING; - } else { - throw new IllegalArgumentException(command + " is not valid for " + this); - } - case NOP: - return this; - default: - throw new IllegalArgumentException("Bad command " + command); - } - } - - public boolean couldHaveIssued(Command command) { - if ((this == State.INSTALLING && command == Command.INSTALL) - || (this == State.STARTING && command == Command.START) - || (this == State.UPGRADING && command == Command.UPGRADE) - || (this == State.STOPPING - && (command == Command.STOP || command == Command.NOP)) - || (this == State.TERMINATING && command == Command.TERMINATE) - ) { - return true; - } - return false; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java deleted file mode 100644 index b6ae4deb46c..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import java.util.ArrayList; -import java.util.List; - -import org.codehaus.jackson.annotate.JsonProperty; - -/** - * Component defined in master package metainfo.json - */ -public abstract class AbstractComponent implements Validate { - public static final String TYPE_STANDARD = "STANDARD"; - public static final String TYPE_DOCKER = "DOCKER"; - public static final String TYPE_PYTHON = "PYTHON"; - public static final String CATEGORY_MASTER = "MASTER"; - public static final String CATEGORY_SLAVE = "SLAVE"; - public static final String CATEGORY_CLIENT = "CLIENT"; - public static final String MASTER_PACKAGE_NAME = "MASTER"; - - protected String name; - protected CommandScript commandScript; - protected List commands = new ArrayList<>(); - - public AbstractComponent() { - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public CommandScript getCommandScript() { - return commandScript; - } - - public void addCommandScript(CommandScript commandScript) { - this.commandScript = commandScript; - } - - @JsonProperty("commands") - public List getCommands() { - return commands; - } - - public void setCommands(List commands) { - this.commands = commands; - } - - public void addCommand(ComponentCommand command) { - commands.add(command); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("{"); - sb.append("\n\"name\": ").append(name); - sb.append(",\n\"commandScript\" :").append(commandScript); - sb.append('}'); - return sb.toString(); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java deleted file mode 100644 index 67d1f156731..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - -import org.apache.commons.digester.Digester; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.SAXException; - -import java.io.IOException; -import java.io.InputStream; -import java.io.StringWriter; - -/** - * This abstract class provide common functionality to parse metainfo.json for - * either master package or add on packages. - */ -public abstract class AbstractMetainfoParser { - protected final GsonBuilder gsonBuilder = new GsonBuilder(); - protected final Gson gson; - private static final Logger log = LoggerFactory - .getLogger(AbstractMetainfoParser.class); - - public AbstractMetainfoParser() { - gson = gsonBuilder.create(); - } - - /** - * Convert to a JSON string - * - * @return a JSON string description - * - * @throws IOException Problems mapping/writing the object - */ - public String toJsonString(Metainfo metaInfo) throws IOException { - return gson.toJson(metaInfo); - } - - /** - * Convert from JSON - * - * @param json input - * - * @return the parsed JSON - * - * @throws IOException IO - */ - public Metainfo fromJsonString(String json) - throws IOException { - return gson.fromJson(json, Metainfo.class); - } - - /** - * Parse metainfo from an IOStream - * - * @param is - * - * @return - * - * @throws IOException - */ - public Metainfo fromJsonStream(InputStream is) throws IOException { - log.debug("loading from xml stream"); - StringWriter writer = new StringWriter(); - IOUtils.copy(is, writer); - return fromJsonString(writer.toString()); - } - - /** - * Parse metainfo from an XML formatted IOStream - * - * @param metainfoStream - * - * @return - * - * @throws IOException - */ - public Metainfo fromXmlStream(InputStream metainfoStream) throws IOException { - log.debug("loading from xml stream"); - Digester digester = new Digester(); - digester.setValidating(false); - - composeSchema(digester); - - try { - return (Metainfo) digester.parse(metainfoStream); - } catch (IOException e) { - log.debug("IOException in metainfoparser during fromXmlStream: " - + e.getMessage()); - } catch (SAXException e) { - log.debug("SAXException in metainfoparser during fromXmlStream: " - + e.getMessage()); - } finally { - if (metainfoStream != null) { - metainfoStream.close(); - } - } - - return null; - } - - /** - * Compose the schema for the metainfo - * - * @param Digester - The Digester object we passed in to compose the schema - * - * @return - * - * @throws IOException - */ - abstract protected void composeSchema(Digester digester); -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java deleted file mode 100644 index cfa2895b143..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.codehaus.jackson.annotate.JsonProperty; - -import java.util.ArrayList; -import java.util.List; - -/** - * Application type defined in the metainfo - */ -public abstract class AbstractMetainfoSchema implements Validate { - protected String name; - protected String comment; - protected String version; - protected List configFiles = new ArrayList<>(); - - public AbstractMetainfoSchema() { - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getComment() { - return comment; - } - - public void setComment(String comment) { - this.comment = comment; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - public void addConfigFile(ConfigFile configFile) { - this.configFiles.add(configFile); - } - - @JsonProperty("configFiles") - public List getConfigFiles() { - return configFiles; - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java deleted file mode 100644 index c75837f16ce..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.commons.digester.Digester; - -/** - * - */ -public class AddonPackageMetainfoParser extends AbstractMetainfoParser { - - protected void composeSchema(Digester digester) { - digester.addObjectCreate("metainfo", Metainfo.class); - digester.addBeanPropertySetter("metainfo/schemaVersion"); - - digester.addObjectCreate("*/applicationPackage", ApplicationPackage.class); - digester.addBeanPropertySetter("*/applicationPackage/name"); - digester.addBeanPropertySetter("*/applicationPackage/comment"); - digester.addBeanPropertySetter("*/applicationPackage/version"); - - digester.addObjectCreate("*/component", ComponentsInAddonPackage.class); - digester.addBeanPropertySetter("*/component/name"); - digester.addSetNext("*/component", "addComponent"); - - digester.addObjectCreate("*/commandScript", CommandScript.class); - digester.addBeanPropertySetter("*/commandScript/script"); - digester.addBeanPropertySetter("*/commandScript/scriptType"); - digester.addBeanPropertySetter("*/commandScript/timeout"); - digester.addSetNext("*/commandScript", "addCommandScript"); - - digester.addObjectCreate("*/configFile", ConfigFile.class); - digester.addBeanPropertySetter("*/configFile/type"); - digester.addBeanPropertySetter("*/configFile/fileName"); - digester.addBeanPropertySetter("*/configFile/dictionaryName"); - digester.addSetNext("*/configFile", "addConfigFile"); - - digester.addSetRoot("*/applicationPackage", "setApplicationPackage"); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java deleted file mode 100644 index 5556c7f5b55..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.BadCommandArgumentsException; -import org.apache.slider.core.exceptions.SliderException; -import org.codehaus.jackson.annotate.JsonIgnore; -import org.codehaus.jackson.annotate.JsonProperty; - -import java.util.ArrayList; -import java.util.List; - -/** - * Application type defined in the metainfo - */ -public class Application extends AbstractMetainfoSchema { - String exportedConfigs; - List exportGroups = new ArrayList<>(); - List osSpecifics = new ArrayList<>(); - List commandOrders = new ArrayList<>(); - List packages = new ArrayList<>(); - private List components = new ArrayList<>(); - - public Application() { - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getComment() { - return comment; - } - - public void setComment(String comment) { - this.comment = comment; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - public String getExportedConfigs() { - return exportedConfigs; - } - - public void setExportedConfigs(String exportedConfigs) { - this.exportedConfigs = exportedConfigs; - } - - public void addConfigFile(ConfigFile configFile) { - this.configFiles.add(configFile); - } - - @JsonProperty("configFiles") - public List getConfigFiles() { - return configFiles; - } - - public void addComponent(Component component) { - components.add(component); - } - - @JsonProperty("components") - public List getComponents() { - return components; - } - - public void addExportGroup(ExportGroup exportGroup) { - exportGroups.add(exportGroup); - } - - @JsonProperty("exportGroups") - public List getExportGroups() { - return exportGroups; - } - - public void addOSSpecific(OSSpecific osSpecific) { - osSpecifics.add(osSpecific); - } - - @JsonIgnore - public List getOSSpecifics() { - return osSpecifics; - } - - public void addCommandOrder(CommandOrder commandOrder) { - commandOrders.add(commandOrder); - } - - @JsonProperty("commandOrders") - public List getCommandOrders() { - return commandOrders; - } - - public void addPackage(Package pkg) { - packages.add(pkg); - } - - @JsonProperty("packages") - public List getPackages() { - return packages; - } - - @Override - public String toString() { - final StringBuilder sb = - new StringBuilder("{"); - sb.append(",\n\"name\": ").append(name); - sb.append(",\n\"comment\": ").append(comment); - sb.append(",\n\"version\" :").append(version); - sb.append(",\n\"components\" : {"); - for (Component component : components) { - sb.append("\n").append(component.toString()); - } - sb.append("\n},"); - sb.append('}'); - return sb.toString(); - } - - public void validate(String version) throws SliderException { - if(SliderUtils.isUnset(version)) { - throw new BadCommandArgumentsException("schema version cannot be null"); - } - - Metainfo.checkNonNull(getName(), "name", "application"); - - Metainfo.checkNonNull(getVersion(), "version", "application"); - - if(getComponents().size() == 0) { - throw new SliderException("application must contain at least one component"); - } - - if(version.equals(Metainfo.VERSION_TWO_ZERO)) { - if(getPackages().size() > 0) { - throw new SliderException("packages is not supported in version " + version); - } - } - - if(version.equals(Metainfo.VERSION_TWO_ONE)) { - if(getOSSpecifics().size() > 0) { - throw new SliderException("osSpecifics is not supported in version " + version); - } - } - - for(CommandOrder co : getCommandOrders()) { - co.validate(version); - } - - for(Component comp : getComponents()) { - comp.validate(version); - } - - for(ConfigFile cf : getConfigFiles()) { - cf.validate(version); - } - - for(ExportGroup eg : getExportGroups()) { - eg.validate(version); - } - - for(Package pkg : getPackages()) { - pkg.validate(version); - } - - for(OSSpecific os : getOSSpecifics()) { - os.validate(version); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java deleted file mode 100644 index a94a2130eee..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.slider.core.exceptions.SliderException; - -public class ApplicationPackage extends AbstractMetainfoSchema{ - private List components = new ArrayList(); - - public void addComponent(ComponentsInAddonPackage component) { - components.add(component); - } - - // we must override getcomponent() as well. otherwise it is pointing to the - // overriden components of type List - public List getComponents(){ - return this.components; - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("{"); - sb.append("\n\"name\": ").append(name); - sb.append(",\n\"comment\": ").append(comment); - sb.append(",\n\"version\" :").append(version); - sb.append(",\n\"components\" : {"); - for (ComponentsInAddonPackage component : components) { - sb.append("\n").append(component); - } - sb.append("\n},"); - sb.append('}'); - return sb.toString(); - } - - @Override - public void validate(String version) throws SliderException { - if (name == null || name.isEmpty()) { - throw new SliderException( - "Missing name in metainfo.json for add on packages"); - } - if (components.isEmpty()) { - throw new SliderException( - "Missing components in metainfo.json for add on packages"); - } - for (ComponentsInAddonPackage component : components) { - if (component.name == null || component.name.isEmpty()) { - throw new SliderException( - "Missing name of components in metainfo.json for add on packages"); - } - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java deleted file mode 100644 index 40d8cc65828..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.core.exceptions.SliderException; - -/** - * - */ -public class CommandOrder implements Validate { - String command; - String requires; - - public CommandOrder() { - } - - public String getCommand() { - return command; - } - - public void setCommand(String command) { - this.command = command; - } - - public String getRequires() { - return requires; - } - - public void setRequires(String requires) { - this.requires = requires; - } - - @Override - public String toString() { - final StringBuilder sb = - new StringBuilder("{"); - sb.append(",\n\"command\": ").append(command); - sb.append(",\n\"requires\": ").append(requires); - sb.append('}'); - return sb.toString(); - } - - public void validate(String version) throws SliderException { - Metainfo.checkNonNull(getCommand(), "command", "package"); - Metainfo.checkNonNull(getRequires(), "requires", "package"); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java deleted file mode 100644 index 9915ba1afc5..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.core.exceptions.SliderException; - -/** - * CommandScript that implements all component commands - */ -public class CommandScript implements Validate { - String script; - String scriptType; - long timeout; - - public CommandScript() { - - } - - public String getScript() { - return script; - } - - public void setScript(String script) { - this.script = script; - } - - public String getScriptType() { - return scriptType; - } - - public void setScriptType(String scriptType) { - this.scriptType = scriptType; - } - - public long getTimeout() { - return timeout; - } - - public void setTimeout(long timeout) { - this.timeout = timeout; - } - - @Override - public String toString() { - final StringBuilder sb = - new StringBuilder("{"); - sb.append(",\n\"script\": ").append(script); - sb.append(",\n\"scriptType\": ").append(scriptType); - sb.append(",\n\"timeout\" :").append(timeout); - sb.append('}'); - return sb.toString(); - } - - public void validate(String version) throws SliderException { - Metainfo.checkNonNull(getScript(), "script", "commandScript"); - Metainfo.checkNonNull(getScriptType(), "scriptType", "commandScript"); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java deleted file mode 100644 index 78bb8c1c005..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.core.exceptions.SliderException; -import org.codehaus.jackson.annotate.JsonProperty; -import java.util.ArrayList; -import java.util.List; - -/** - * Component defined in master package metainfo.json - */ -public class Component extends AbstractComponent { - - String category = CATEGORY_MASTER; - String publishConfig = Boolean.FALSE.toString(); - String minInstanceCount = "0"; - String maxInstanceCount; - String autoStartOnFailure = Boolean.FALSE.toString(); - String appExports; - String compExports; - String type = TYPE_STANDARD; - List componentExports = new ArrayList<>(); - List dockerContainers = new ArrayList<>(); - List configFiles = new ArrayList<>(); - - public Component() { - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getCategory() { - return category; - } - - public void setCategory(String category) { - this.category = category; - } - - public String getPublishConfig() { - return publishConfig; - } - - public void setPublishConfig(String publishConfig) { - this.publishConfig = publishConfig; - } - - public String getAutoStartOnFailure() { - return autoStartOnFailure; - } - - public void setAutoStartOnFailure(String autoStartOnFailure) { - this.autoStartOnFailure = autoStartOnFailure; - } - - public String getAppExports() { - return appExports; - } - - public void setAppExports(String appExports) { - this.appExports = appExports; - } - - public String getCompExports() { - return compExports; - } - - public void setCompExports(String compExports) { - this.compExports = compExports; - } - - public String getMinInstanceCount() { - return minInstanceCount; - } - - @JsonProperty("dockerContainers") - public List getDockerContainers() { - return this.dockerContainers; - } - - public Boolean getAutoStartOnFailureBoolean() { - if (SliderUtils.isUnset(getAutoStartOnFailure())) { - return Boolean.FALSE; - } - - return Boolean.parseBoolean(getAutoStartOnFailure()); - } - - public int getMinInstanceCountInt() throws BadConfigException { - if (SliderUtils.isUnset(minInstanceCount)) { - return 0; - } - - try { - return Integer.parseInt(minInstanceCount); - } catch (NumberFormatException nfe) { - throw new BadConfigException(nfe, "Invalid value for minInstanceCount for %s", name); - } - } - - public int getMaxInstanceCountInt() throws BadConfigException { - if (SliderUtils.isUnset(maxInstanceCount)) { - return Integer.MAX_VALUE; - } - - try { - return Integer.parseInt(maxInstanceCount); - } catch (NumberFormatException nfe) { - throw new BadConfigException(nfe, "Invalid value for maxInstanceCount for %s", name); - } - } - - public void setMinInstanceCount(String minInstanceCount) { - this.minInstanceCount = minInstanceCount; - } - - public String getMaxInstanceCount() { - return maxInstanceCount; - } - - public void setMaxInstanceCount(String maxInstanceCount) { - this.maxInstanceCount = maxInstanceCount; - } - - public void addComponentExport(ComponentExport export) { - componentExports.add(export); - } - - public List getComponentExports() { - return componentExports; - } - - public Boolean getRequiresAutoRestart() { - return Boolean.parseBoolean(this.autoStartOnFailure); - } - - public void addConfigFile(ConfigFile configFile) { - this.configFiles.add(configFile); - } - - @JsonProperty("configFiles") - public List getConfigFiles() { - return configFiles; - } - - @Override - public String toString() { - final StringBuilder sb = - new StringBuilder("{"); - sb.append("\n\"name\": ").append(name); - sb.append(",\n\"category\": ").append(category); - sb.append(",\n\"commandScript\" :").append(commandScript); - for(DockerContainer dc : dockerContainers){ - sb.append(",\n\"container\" :").append(dc.toString()); - } - sb.append('}'); - return sb.toString(); - } - - public void validate(String version) throws SliderException { - Metainfo.checkNonNull(getName(), "name", "component"); - Metainfo.checkNonNull(getCategory(), "category", "component"); - if (!getCategory().equals(CATEGORY_MASTER) - && !getCategory().equals(CATEGORY_SLAVE) - && !getCategory().equals(CATEGORY_CLIENT)) { - throw new SliderException("Invalid category for the component " + getCategory()); - } - - Metainfo.checkNonNull(getType(), "type", "component"); - if (!getType().equals(TYPE_DOCKER) - && !getType().equals(TYPE_STANDARD)) { - throw new SliderException("Invalid type for the component " + getType()); - } - - if (version.equals(Metainfo.VERSION_TWO_ZERO)) { - if (getType().equals(TYPE_DOCKER)) { - throw new SliderException(TYPE_DOCKER + " is not supported in version " + Metainfo.VERSION_TWO_ZERO); - } - - if (getCommands().size() > 0) { - throw new SliderException("commands are not supported in version " + Metainfo.VERSION_TWO_ZERO); - } - } - - if (commandScript != null) { - commandScript.validate(version); - } - - if (version.equals(Metainfo.VERSION_TWO_ONE)) { - for (ComponentCommand cc : getCommands()) { - cc.validate(version); - } - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java deleted file mode 100644 index 52117c530e3..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.core.exceptions.SliderException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Represents the metadata associated with the application. - */ -public class ComponentCommand implements Validate { - protected static final Logger - log = LoggerFactory.getLogger(ComponentCommand.class); - - - private String exec; - private String name = "START"; - private String type = "SHELL"; - - /** - * Creator. - */ - public ComponentCommand() { - } - - public void setName(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public void setExec(String exec) { - this.exec = exec; - } - - public String getExec() { - return exec; - } - - public void setType(String type) { - this.type = type; - } - - public String getType() { - return type; - } - - public void validate(String version) throws SliderException { - Metainfo.checkNonNull(getName(), "name", "componentCommand"); - - Metainfo.checkNonNull(getType(), "version", "application"); - } - - public static ComponentCommand getDefaultComponentCommand() { - ComponentCommand cc = new ComponentCommand(); - cc.setExec("DEFAULT"); - return cc; - } - - public static ComponentCommand getDefaultComponentCommand(String commandName) { - ComponentCommand cc = new ComponentCommand(); - cc.setExec("DEFAULT"); - cc.setName(commandName); - return cc; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java deleted file mode 100644 index a18854c79a2..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -/** - * - */ -public class ComponentExport { - String name; - String value; - - public ComponentExport() { - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - @Override - public String toString() { - final StringBuilder sb = - new StringBuilder("{"); - sb.append(",\n\"name\": ").append(name); - sb.append(",\n\"value\": ").append(value); - sb.append('}'); - return sb.toString(); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java deleted file mode 100644 index 855e5b6b33e..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.core.exceptions.SliderException; - -public class ComponentsInAddonPackage extends AbstractComponent { - - @Override - public void validate(String version) throws SliderException { - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java deleted file mode 100644 index cb47512f8a8..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.core.exceptions.SliderException; - -/** - * - */ -public class ConfigFile implements Validate { - String type; - String fileName; - String dictionaryName; - - public ConfigFile() { - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getFileName() { - return fileName; - } - - public void setFileName(String fileName) { - this.fileName = fileName; - } - - public String getDictionaryName() { - return dictionaryName; - } - - public void setDictionaryName(String dictionaryName) { - this.dictionaryName = dictionaryName; - } - - public void validate(String version) throws SliderException { - - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java deleted file mode 100644 index 46c8836ebea..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import java.util.ArrayList; -import java.util.List; - -/** - * Application default config - */ -public class DefaultConfig { - List propertyInfos; - - public DefaultConfig() { - propertyInfos = new ArrayList(); - } - - public void addPropertyInfo(PropertyInfo propertyInfo) { - propertyInfos.add(propertyInfo); - } - - public List getPropertyInfos() { - return propertyInfos; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java deleted file mode 100644 index e136775bd2f..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.commons.digester.Digester; -import org.xml.sax.SAXException; - -import java.io.IOException; -import java.io.InputStream; - -/** - * - */ -public class DefaultConfigParser { - - public DefaultConfig parse(InputStream configFileStream) throws IOException { - Digester digester = new Digester(); - digester.setValidating(false); - - digester.addObjectCreate("configuration", DefaultConfig.class); - - digester.addObjectCreate("*/property", PropertyInfo.class); - digester.addBeanPropertySetter("*/property/name"); - digester.addBeanPropertySetter("*/property/value"); - digester.addBeanPropertySetter("*/property/description"); - digester.addSetNext("*/property", "addPropertyInfo"); - - try { - return (DefaultConfig) digester.parse(configFileStream); - } catch (IOException e) { - - } catch (SAXException e) { - - } finally { - configFileStream.close(); - } - - return null; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java deleted file mode 100644 index 4c61e7f1bb7..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.core.exceptions.SliderException; -import org.codehaus.jackson.annotate.JsonProperty; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -/** - * Represents a docker container - */ -public class DockerContainer implements Validate { - protected static final Logger - log = LoggerFactory.getLogger(DockerContainer.class); - - private String name; - private String image; - private String network; - private String useNetworkScript; - private String options; - private List mounts = new ArrayList<>(); - private List ports = new ArrayList<>(); - private String statusCommand; - private String startCommand; - private String commandPath; - private String additionalParam; - private String runPrivilegedContainer; - private List inputFiles = new ArrayList<>(); - private List configFiles = new ArrayList<>(); - - public DockerContainer() { - } - - @JsonProperty("mounts") - public List getMounts() { return this.mounts; } - - @JsonProperty("ports") - public List getPorts() { - return this.ports; - } - - @JsonProperty("inputFiles") - public List getInputFiles() { - return this.inputFiles; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getImage() { - return image; - } - - public void setImage(String image) { - this.image = image; - } - - public String getNetwork() { - return network; - } - - public void setNetwork(String network) { - this.network = network; - } - - public String getUseNetworkScript() { - return useNetworkScript; - } - - public void setUseNetworkScript(String useNetworkScript) { - this.useNetworkScript = useNetworkScript; - } - - public String getOptions() { - return options; - } - - public void setOptions(String options) { - this.options = options; - } - - @Override - public void validate(String version) throws SliderException { - Metainfo.checkNonNull(getName(), "name", "dockerContainer"); - Metainfo.checkNonNull(getImage(), "image", "dockerContainer"); - for (DockerContainerMount dcm : getMounts()) { - dcm.validate(version); - } - for (DockerContainerPort dcp : getPorts()) { - dcp.validate(version); - } - } - - @JsonProperty("statusCommand") - public String getStatusCommand() { - return statusCommand; - } - - @JsonProperty("statusCommand") - public void setStatusCommand(String statusCommand) { - this.statusCommand = statusCommand; - } - - public String getCommandPath() { - return commandPath; - } - - public void setCommandPath(String commandPath) { - this.commandPath = commandPath; - } - - public String getAdditionalParam() { - return additionalParam; - } - - public void setAdditionalParam(String additionalParam) { - this.additionalParam = additionalParam; - } - - @JsonProperty("startCommand") - public String getStartCommand() { - return startCommand; - } - - @JsonProperty("startCommand") - public void setStartCommand(String startCommand) { - this.startCommand = startCommand; - } - - @JsonProperty("runPrivilegedContainer") - public String getRunPrivilegedContainer() { - return runPrivilegedContainer; - } - - @JsonProperty("runPrivilegedContainer") - public void setRunPrivilegedContainer(String runPrivilegedContainer) { - this.runPrivilegedContainer = runPrivilegedContainer; - } - - public List getConfigFiles() { - return configFiles; - } - - public void setConfigFiles(List configFiles) { - this.configFiles = configFiles; - } - - @Override - public String toString() { - StringBuilder result = new StringBuilder("DockerContainer [name=") - .append(name).append(", image=").append(image).append(", options=") - .append(options).append(", mounts=").append(mounts).append(", ports=") - .append(ports).append(", statusCommand=").append(statusCommand) - .append(", commandPath=").append(commandPath) - .append(", additionalParam=").append(additionalParam) - .append(", inputFiles=").append(inputFiles).append(", startCommand=") - .append(startCommand).append(", runPriviledgedContainer=") - .append(runPrivilegedContainer).append(", configFiles=") - .append(configFiles).append("]"); - return result.toString(); - } -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java deleted file mode 100644 index 0faceb96c1e..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent.application.metadata; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DockerContainerInputFile { - protected static final Logger log = LoggerFactory - .getLogger(DockerContainerInputFile.class); - - private String containerPath; - private String fileLocalPath; - - public DockerContainerInputFile() { - } - - public String getContainerMount() { - return containerPath; - } - - public void setContainerMount(String containerMount) { - this.containerPath = containerMount; - } - - public String getFileLocalPath() { - return fileLocalPath; - } - - public void setFileLocalPath(String fileLocalPath) { - this.fileLocalPath = fileLocalPath; - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java deleted file mode 100644 index 61f07f41608..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.core.exceptions.SliderException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Represents a docker container mount - */ -public class DockerContainerMount implements Validate { - protected static final Logger - log = LoggerFactory.getLogger(DockerContainerMount.class); - - - private String containerMount; - private String hostMount; - - public DockerContainerMount() { - } - - public String getContainerMount() { - return containerMount; - } - - public void setContainerMount(String containerMount) { - this.containerMount = containerMount; - } - - public String getHostMount() { - return hostMount; - } - - public void setHostMount(String hostMount) { - this.hostMount = hostMount; - } - - @Override - public void validate(String version) throws SliderException { - Metainfo.checkNonNull(getContainerMount(), "containerMount", "dockerContainerMount"); - Metainfo.checkNonNull(getHostMount(), "hostMount", "dockerContainerMount"); - } -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java deleted file mode 100644 index 0629d9d3e3f..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.core.exceptions.SliderException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Represents a docker container port - */ -public class DockerContainerPort implements Validate { - protected static final Logger - log = LoggerFactory.getLogger(DockerContainerPort.class); - - - private String containerPort; - private String hostPort; - - public DockerContainerPort() { - } - - public String getContainerPort() { - return containerPort; - } - - public void setContainerPort(String containerPort) { - this.containerPort = containerPort; - } - - public String getHostPort() { - return hostPort; - } - - public void setHostPort(String hostPort) { - this.hostPort = hostPort; - } - - @Override - public void validate(String version) throws SliderException { - Metainfo.checkNonNull(getContainerPort(), "containerPort", "dockerContainerPort"); - Metainfo.checkNonNull(getHostPort(), "hostPort", "dockerContainerPort"); - } - - @Override - public String toString() { - return "DockerContainerPort [containerPort=" + containerPort - + ", hostPort=" + hostPort + "]"; - } -} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java deleted file mode 100644 index 5e0fb24581f..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.core.exceptions.SliderException; - -/** - * - */ -public class Export implements Validate { - String name; - String value; - - public Export() { - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - @Override - public String toString() { - final StringBuilder sb = - new StringBuilder("{"); - sb.append(",\n\"name\": ").append(name); - sb.append(",\n\"value\": ").append(value); - sb.append('}'); - return sb.toString(); - } - - public void validate(String version) throws SliderException { - Metainfo.checkNonNull(getName(), "name", "export"); - Metainfo.checkNonNull(getValue(), "value", "export"); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java deleted file mode 100644 index 3d9f34c685d..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.core.exceptions.SliderException; - -import java.util.ArrayList; -import java.util.List; - -/** - * - */ -public class ExportGroup implements Validate { - String name; - List exports; - - public ExportGroup() { - exports = new ArrayList(); - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public void addExport(Export export) { - exports.add(export); - } - - public List getExports() { - return exports; - } - - @Override - public String toString() { - final StringBuilder sb = - new StringBuilder("{"); - sb.append(",\n\"name\": ").append(name); - sb.append(",\n\"exports\" : {"); - for (Export export : exports) { - sb.append("\n").append(export); - } - sb.append("\n},"); - sb.append('}'); - return sb.toString(); - } - - public void validate(String version) throws SliderException { - Metainfo.checkNonNull(getName(), "name", "exportGroup"); - for(Export exp : getExports()) { - exp.validate(version); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java deleted file mode 100644 index 10c497fcbc3..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.common.tools.SliderUtils; -import org.apache.slider.core.exceptions.SliderException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -/** - * Application metainfo uber class - */ -public class Metainfo { - protected static final Logger log = - LoggerFactory.getLogger(Metainfo.class); - public static String VERSION_TWO_ZERO = "2.0"; - public static String VERSION_TWO_ONE = "2.1"; - - String schemaVersion; - ApplicationPackage applicationPackage; - Application application; - - public String getSchemaVersion() { - return schemaVersion; - } - - public void setSchemaVersion(String schemaVersion) { - this.schemaVersion = schemaVersion; - } - - public ApplicationPackage getApplicationPackage() { - return applicationPackage; - } - - public void setApplicationPackage(ApplicationPackage pkg) { - this.applicationPackage = pkg; - } - - public Application getApplication() { - return application; - } - - public void setApplication(Application application) { - this.application = application; - } - - public Component getApplicationComponent(String roleGroup) { - if (application == null) { - log.error("Malformed app definition: Expect application as the top level element for metainfo"); - } else { - for (Component component : application.getComponents()) { - if (component.getName().equals(roleGroup)) { - return component; - } - } - } - return null; - } - - public List getComponentConfigFiles(String roleGroup) { - List componentConfigFiles = new ArrayList<>(); - componentConfigFiles.addAll(application.getConfigFiles()); - Component component = getApplicationComponent(roleGroup); - if (component != null) { - componentConfigFiles.addAll(component.getConfigFiles()); - } - return componentConfigFiles; - } - - public void validate() throws SliderException { - if (!VERSION_TWO_ONE.equals(schemaVersion) && - !VERSION_TWO_ZERO.equals(schemaVersion)) { - throw new SliderException("Unsupported version " + getSchemaVersion()); - } - if (application != null) { - application.validate(schemaVersion); - } - if (applicationPackage != null) { - applicationPackage.validate(schemaVersion); - } - } - - public static void checkNonNull(String value, String field, String type) throws SliderException { - if (SliderUtils.isUnset(value)) { - throw new SliderException(type + "." + field + " cannot be null"); - } - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("Metainfo [schemaVersion="); - builder.append(schemaVersion); - builder.append(", applicationPackage="); - builder.append(applicationPackage); - builder.append(", application="); - builder.append(application); - builder.append("]"); - return builder.toString(); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java deleted file mode 100644 index 8b520eb6c90..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.commons.digester.Digester; - -/** - * - */ -public class MetainfoParser extends AbstractMetainfoParser{ - - protected void composeSchema(Digester digester){ - digester.addObjectCreate("metainfo", Metainfo.class); - digester.addBeanPropertySetter("metainfo/schemaVersion"); - - digester.addObjectCreate("*/application", Application.class); - digester.addBeanPropertySetter("*/application/name"); - digester.addBeanPropertySetter("*/application/comment"); - digester.addBeanPropertySetter("*/application/version"); - digester.addBeanPropertySetter("*/application/exportedConfigs"); - - digester.addObjectCreate("*/commandOrder", CommandOrder.class); - digester.addBeanPropertySetter("*/commandOrder/command"); - digester.addBeanPropertySetter("*/commandOrder/requires"); - digester.addSetNext("*/commandOrder", "addCommandOrder"); - - digester.addObjectCreate("*/exportGroup", ExportGroup.class); - digester.addBeanPropertySetter("*/exportGroup/name"); - digester.addObjectCreate("*/export", Export.class); - digester.addBeanPropertySetter("*/export/name"); - digester.addBeanPropertySetter("*/export/value"); - digester.addSetNext("*/export", "addExport"); - digester.addSetNext("*/exportGroup", "addExportGroup"); - - digester.addObjectCreate("*/component", Component.class); - digester.addBeanPropertySetter("*/component/name"); - digester.addBeanPropertySetter("*/component/category"); - digester.addBeanPropertySetter("*/component/publishConfig"); - digester.addBeanPropertySetter("*/component/minInstanceCount"); - digester.addBeanPropertySetter("*/component/maxInstanceCount"); - digester.addBeanPropertySetter("*/component/autoStartOnFailure"); - digester.addBeanPropertySetter("*/component/appExports"); - digester.addBeanPropertySetter("*/component/compExports"); - digester.addObjectCreate("*/componentExport", ComponentExport.class); - digester.addBeanPropertySetter("*/componentExport/name"); - digester.addBeanPropertySetter("*/componentExport/value"); - digester.addSetNext("*/componentExport", "addComponentExport"); - digester.addSetNext("*/component", "addComponent"); - - digester.addObjectCreate("*/commandScript", CommandScript.class); - digester.addBeanPropertySetter("*/commandScript/script"); - digester.addBeanPropertySetter("*/commandScript/scriptType"); - digester.addBeanPropertySetter("*/commandScript/timeout"); - digester.addSetNext("*/commandScript", "addCommandScript"); - - digester.addObjectCreate("*/command", ComponentCommand.class); - digester.addBeanPropertySetter("*/command/exec"); - digester.addBeanPropertySetter("*/command/name"); - digester.addBeanPropertySetter("*/command/type"); - digester.addSetNext("*/command", "addCommand"); - - digester.addObjectCreate("*/osSpecific", OSSpecific.class); - digester.addBeanPropertySetter("*/osSpecific/osType"); - digester.addObjectCreate("*/osSpecific/packages/package", OSPackage.class); - digester.addBeanPropertySetter("*/osSpecific/packages/package/type"); - digester.addBeanPropertySetter("*/osSpecific/packages/package/name"); - digester.addSetNext("*/osSpecific/packages/package", "addOSPackage"); - digester.addSetNext("*/osSpecific", "addOSSpecific"); - - digester.addObjectCreate("*/application/packages/package", Package.class); - digester.addBeanPropertySetter("*/application/packages/package/type"); - digester.addBeanPropertySetter("*/application/packages/package/name"); - digester.addSetNext("*/application/packages/package", "addPackage"); - - digester.addObjectCreate("*/configFile", ConfigFile.class); - digester.addBeanPropertySetter("*/configFile/type"); - digester.addBeanPropertySetter("*/configFile/fileName"); - digester.addBeanPropertySetter("*/configFile/dictionaryName"); - digester.addSetNext("*/configFile", "addConfigFile"); - - digester.addSetRoot("*/application", "setApplication"); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java deleted file mode 100644 index 32b48903705..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.core.exceptions.SliderException; - -/** - * - */ -public class OSPackage implements Validate { - String type; - String name; - - public OSPackage() { - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public void validate(String version) throws SliderException { - Metainfo.checkNonNull(getName(), "name", "osPackage"); - Metainfo.checkNonNull(getType(), "type", "osPackage"); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java deleted file mode 100644 index c06d4982d44..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.core.exceptions.SliderException; - -import java.util.ArrayList; -import java.util.List; - -/** - * - */ -public class OSSpecific implements Validate { - String osType; - List packages; - - public OSSpecific() { - packages = new ArrayList(); - } - - public String getOsType() { - return osType; - } - - public void setOsType(String osType) { - this.osType = osType; - } - - public void addOSPackage(OSPackage osPackage) { - packages.add(osPackage); - } - - public List getPackages() { - return packages; - } - - public void validate(String version) throws SliderException { - Metainfo.checkNonNull(getOsType(), "osType", "osSpecific"); - for (OSPackage opkg : getPackages()) { - opkg.validate(version); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java deleted file mode 100644 index b88f77d1831..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.core.exceptions.SliderException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Represents package description. - */ -public class Package implements Validate { - protected static final Logger - log = LoggerFactory.getLogger(Package.class); - - - private String name; - private String type; - - public Package() { - } - - - public void setName(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public void validate(String version) throws SliderException { - Metainfo.checkNonNull(getName(), "name", "package"); - Metainfo.checkNonNull(getType(), "type", "package"); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java deleted file mode 100644 index 62ee0f50a71..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -/** - * Application config property info - */ -public class PropertyInfo { - String name; - String value; - String description; - - public PropertyInfo() { - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java deleted file mode 100644 index ef03dcdd2f5..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.core.exceptions.SliderException; - -/** - * Implementer provides a validate method - */ -public interface Validate { - - public void validate(String version) throws SliderException; -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md deleted file mode 100644 index dfd137349f1..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md +++ /dev/null @@ -1,22 +0,0 @@ - - -# todo list - -* Retry on failure - * Agent can toleate a configurable number of failures (e.g. 3) before giving up -* Agent should separate out hostname and label that is received for registration diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 57ec2183c40..34b6a7d5856 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -20,12 +20,8 @@ import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.health.HealthCheckRegistry; -import com.codahale.metrics.jvm.GarbageCollectorMetricSet; -import com.codahale.metrics.jvm.MemoryUsageGaugeSet; -import com.codahale.metrics.jvm.ThreadStatesGaugeSet; import com.google.common.base.Preconditions; import com.google.protobuf.BlockingService; - import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -36,8 +32,14 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.io.Text; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; +import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; +import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -66,18 +68,9 @@ import org.apache.hadoop.yarn.client.api.async.NMClientAsync; import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.*; -import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG; - import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.hadoop.registry.client.binding.RegistryPathUtils; -import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; -import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; @@ -122,18 +115,16 @@ import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderService; import org.apache.slider.providers.SliderProviderFactory; -import org.apache.slider.providers.agent.AgentKeys; -import org.apache.slider.providers.agent.AgentProviderService; import org.apache.slider.providers.slideram.SliderAMClientProvider; import org.apache.slider.providers.slideram.SliderAMProviderService; import org.apache.slider.server.appmaster.actions.ActionRegisterServiceInstance; -import org.apache.slider.server.appmaster.actions.EscalateOutstandingRequests; -import org.apache.slider.server.appmaster.actions.RegisterComponentInstance; -import org.apache.slider.server.appmaster.actions.QueueExecutor; -import org.apache.slider.server.appmaster.actions.QueueService; import org.apache.slider.server.appmaster.actions.ActionStopSlider; import org.apache.slider.server.appmaster.actions.ActionUpgradeContainers; import org.apache.slider.server.appmaster.actions.AsyncAction; +import org.apache.slider.server.appmaster.actions.EscalateOutstandingRequests; +import org.apache.slider.server.appmaster.actions.QueueExecutor; +import org.apache.slider.server.appmaster.actions.QueueService; +import org.apache.slider.server.appmaster.actions.RegisterComponentInstance; import org.apache.slider.server.appmaster.actions.RenewingAction; import org.apache.slider.server.appmaster.actions.ResetFailureWindow; import org.apache.slider.server.appmaster.actions.ReviewAndFlexApplicationSize; @@ -143,26 +134,24 @@ import org.apache.slider.server.appmaster.monkey.ChaosKillAM; import org.apache.slider.server.appmaster.monkey.ChaosKillContainer; import org.apache.slider.server.appmaster.monkey.ChaosMonkeyService; +import org.apache.slider.server.appmaster.operations.AbstractRMOperation; import org.apache.slider.server.appmaster.operations.AsyncRMOperationHandler; import org.apache.slider.server.appmaster.operations.ProviderNotifyingOperationHandler; +import org.apache.slider.server.appmaster.operations.RMOperationHandler; import org.apache.slider.server.appmaster.rpc.RpcBinder; import org.apache.slider.server.appmaster.rpc.SliderAMPolicyProvider; import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPBImpl; -import org.apache.slider.server.appmaster.operations.AbstractRMOperation; import org.apache.slider.server.appmaster.rpc.SliderIPCService; import org.apache.slider.server.appmaster.security.SecurityConfiguration; import org.apache.slider.server.appmaster.state.AppState; import org.apache.slider.server.appmaster.state.AppStateBindingInfo; import org.apache.slider.server.appmaster.state.ContainerAssignment; import org.apache.slider.server.appmaster.state.ProviderAppState; -import org.apache.slider.server.appmaster.operations.RMOperationHandler; import org.apache.slider.server.appmaster.state.RoleInstance; -import org.apache.slider.server.appmaster.web.AgentService; -import org.apache.slider.server.appmaster.web.rest.InsecureAmFilterInitializer; -import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp; import org.apache.slider.server.appmaster.web.SliderAMWebApp; import org.apache.slider.server.appmaster.web.WebAppApi; import org.apache.slider.server.appmaster.web.WebAppApiImpl; +import org.apache.slider.server.appmaster.web.rest.InsecureAmFilterInitializer; import org.apache.slider.server.appmaster.web.rest.RestPaths; import org.apache.slider.server.appmaster.web.rest.application.ApplicationResouceContentCacheFactory; import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache; @@ -181,7 +170,6 @@ import java.net.InetSocketAddress; import java.net.URI; import java.net.URL; -import java.net.URLClassLoader; import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; @@ -199,6 +187,9 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.*; +import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG; + /** * This is the AM, which directly implements the callbacks from the AM and NM */ @@ -765,11 +756,6 @@ private int createAndRunCluster(String clustername) throws Throwable { appMasterContainerID.toString(), clustername); certificateManager.setPassphrase(instanceDefinition.getPassphrase()); - - if (component.getOptionBool( - AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false)) { - uploadServerCertForLocalization(clustername, fs); - } // Web service endpoints: initialize WebAppApiImpl webAppApi = @@ -784,10 +770,6 @@ private int createAndRunCluster(String clustername) throws Throwable { contentCache); initAMFilterOptions(serviceConf); - if (providerService instanceof AgentProviderService) { - // start the agent web app - startAgentWebApp(appInformation, serviceConf, webAppApi); - } int webAppPort = deployWebApplication(webAppApi); String scheme = WebAppUtils.HTTP_PREFIX; @@ -1165,26 +1147,6 @@ private int getPortToRequest() throws SliderException, IOException { return portScanner.getAvailablePort(); } - private void uploadServerCertForLocalization(String clustername, - SliderFileSystem fs) - throws IOException { - Path certsDir = fs.buildClusterSecurityDirPath(clustername); - if (!fs.getFileSystem().exists(certsDir)) { - fs.getFileSystem().mkdirs(certsDir, - new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); - } - Path destPath = new Path(certsDir, SliderKeys.CRT_FILE_NAME); - if (!fs.getFileSystem().exists(destPath)) { - fs.getFileSystem().copyFromLocalFile( - new Path(CertificateManager.getServerCertficateFilePath().getAbsolutePath()), - destPath); - log.info("Uploaded server cert to localization path {}", destPath); - } - - fs.getFileSystem().setPermission(destPath, - new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE)); - } - protected void login(String principal, File localKeytabFile) throws IOException, SliderException { log.info("Logging in as {} with keytab {}", principal, localKeytabFile); @@ -1220,53 +1182,6 @@ protected void validateLoginUser(UserGroupInformation user) } } - /** - * Set up and start the agent web application - * @param appInformation application information - * @param serviceConf service configuration - * @param webAppApi web app API instance to bind to - * @throws IOException - */ - private void startAgentWebApp(MapOperations appInformation, - Configuration serviceConf, WebAppApiImpl webAppApi) throws IOException, SliderException { - URL[] urls = ((URLClassLoader) AgentWebApp.class.getClassLoader() ).getURLs(); - StringBuilder sb = new StringBuilder("AM classpath:"); - for (URL url : urls) { - sb.append("\n").append(url.toString()); - } - LOG_YARN.debug(sb.append("\n").toString()); - initAMFilterOptions(serviceConf); - - - // Start up the agent web app and track the URL for it - MapOperations appMasterConfig = getInstanceDefinition() - .getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM); - AgentWebApp agentWebApp = AgentWebApp.$for(AgentWebApp.BASE_PATH, - webAppApi, - RestPaths.AGENT_WS_CONTEXT) - .withComponentConfig(appMasterConfig) - .withPort(getPortToRequest()) - .withSecuredPort(getPortToRequest()) - .start(); - agentOpsUrl = - "https://" + appMasterHostname + ":" + agentWebApp.getSecuredPort(); - agentStatusUrl = - "https://" + appMasterHostname + ":" + agentWebApp.getPort(); - AgentService agentService = - new AgentService("slider-agent", agentWebApp); - - agentService.init(serviceConf); - agentService.start(); - addService(agentService); - - appInformation.put(StatusKeys.INFO_AM_AGENT_OPS_URL, agentOpsUrl + "/"); - appInformation.put(StatusKeys.INFO_AM_AGENT_STATUS_URL, agentStatusUrl + "/"); - appInformation.set(StatusKeys.INFO_AM_AGENT_STATUS_PORT, - agentWebApp.getPort()); - appInformation.set(StatusKeys.INFO_AM_AGENT_OPS_PORT, - agentWebApp.getSecuredPort()); - } - /** * Set up the AM filter * @param serviceConf configuration to patch @@ -1327,17 +1242,8 @@ public void registerServiceInstance(String instanceName, serviceRecord); // provider service dynamic definitions. - if (providerService instanceof AgentProviderService) { - URL agentOpsURI = new URL(agentOpsUrl); - URL agentStatusURI = new URL(agentStatusUrl); - ((AgentProviderService)providerService).applyInitialRegistryDefinitions( - amWebURI, - agentOpsURI, - agentStatusURI, - serviceRecord); - } else { - providerService.applyInitialRegistryDefinitions(amWebURI, serviceRecord); - } + providerService.applyInitialRegistryDefinitions(amWebURI, serviceRecord); + // set any provided attributes setProvidedServiceRecordAttributes( @@ -1800,11 +1706,6 @@ public synchronized void onUpgradeContainers( } LOG_YARN.info("Final list of containers to be upgraded (total {}) : {}", containers.size(), containers); - if (providerService instanceof AgentProviderService) { - AgentProviderService agentProviderService = (AgentProviderService) providerService; - agentProviderService.setInUpgradeMode(true); - agentProviderService.addUpgradeContainers(containers); - } } // create a reverse map of roles -> set of all live containers @@ -1968,11 +1869,6 @@ public void escalateOutstandingRequests() { * Shutdown operation: release all containers */ private void releaseAllContainers() { - if (providerService instanceof AgentProviderService) { - log.info("Setting stopInitiated flag to true"); - AgentProviderService agentProviderService = (AgentProviderService) providerService; - agentProviderService.setAppStopInitiated(true); - } // Add the sleep here (before releasing containers) so that applications get // time to perform graceful shutdown try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java deleted file mode 100644 index f8400355da2..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web; - -import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp; -import org.apache.slider.server.services.workflow.ClosingService; -import org.apache.slider.server.services.workflow.WorkflowCompositeService; - -/** - * agent service gives the agent webapp lifecycle integration - */ -public class AgentService extends ClosingService { - - - public AgentService(String name) { - super(name); - } - - public AgentService(String name, AgentWebApp app) { - super(name, app); - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java index 0f99d6d88bd..65a35918f41 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java @@ -17,7 +17,6 @@ package org.apache.slider.server.appmaster.web; import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.slider.api.SliderClusterProtocol; import org.apache.slider.providers.ProviderService; import org.apache.slider.server.appmaster.AppMasterActionOperations; import org.apache.slider.server.appmaster.actions.QueueAccess; @@ -25,7 +24,6 @@ import org.apache.slider.server.appmaster.state.AppState; import org.apache.slider.server.appmaster.state.RoleStatus; import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations; import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache; import org.apache.slider.server.services.security.CertificateManager; @@ -57,11 +55,6 @@ public interface WebAppApi { * is a computed value and not just a getter */ Map getRoleStatusByName(); - - /** - * Returns an interface that can support the agent-based REST operations. - */ - AgentRestOperations getAgentRestOperations(); /** * Registry operations accessor diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java index a0fe3108591..bd4d2bf1119 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java @@ -23,7 +23,6 @@ import org.apache.slider.server.appmaster.management.MetricsAndMonitoring; import org.apache.slider.server.appmaster.state.RoleStatus; import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations; import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache; import org.apache.slider.server.services.security.CertificateManager; import org.slf4j.Logger; @@ -96,11 +95,6 @@ public Map getRoleStatusByName() { return map; } - @Override - public AgentRestOperations getAgentRestOperations() { - return provider.getAgentRestOperations(); - } - @Override public RegistryOperations getRegistryOperations() { return registryOperations; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java index 03bf703c266..aed87d8455e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java @@ -19,7 +19,6 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import org.apache.slider.server.appmaster.web.WebAppApi; -import org.apache.slider.server.appmaster.web.rest.agent.AgentResource; import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource; import org.apache.slider.server.appmaster.web.rest.management.ManagementResource; import org.apache.slider.server.appmaster.web.rest.publisher.PublisherResource; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java deleted file mode 100644 index 17cd8f2af0a..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -public enum AgentCommandType { - EXECUTION_COMMAND, - STATUS_COMMAND, - REGISTRATION_COMMAND -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java deleted file mode 100644 index 781ae0088ff..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java +++ /dev/null @@ -1,376 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -import com.google.gson.annotations.SerializedName; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class AgentEnv { - - /** - * Various directories, configurable in ambari-agent.ini - */ - private Directory[] stackFoldersAndFiles = new Directory[0]; - - /** - * Directories that match name /etc/alternatives/*conf - */ - private Alternative[] alternatives = new Alternative[0]; - - /** - * List of existing users - */ - private ExistingUser[] existingUsers = new ExistingUser[0]; - - /** - * List of repos - */ - private String[] existingRepos = new String[0]; - - /** - * List of packages - */ - private PackageDetail[] installedPackages = new PackageDetail[0]; - - /** - * The host health report - */ - private HostHealth hostHealth = new HostHealth(); - - private Integer umask; - - private Boolean iptablesIsRunning; - - public Integer getUmask() { - return umask; - } - - public void setUmask(Integer umask) { - this.umask = umask; - } - - public Directory[] getStackFoldersAndFiles() { - return stackFoldersAndFiles; - } - - public void setStackFoldersAndFiles(Directory[] dirs) { - stackFoldersAndFiles = dirs; - } - - public void setExistingUsers(ExistingUser[] users) { - existingUsers = users; - } - - public ExistingUser[] getExistingUsers() { - return existingUsers; - } - - public void setAlternatives(Alternative[] dirs) { - alternatives = dirs; - } - - public Alternative[] getAlternatives() { - return alternatives; - } - - public void setExistingRepos(String[] repos) { - existingRepos = repos; - } - - public String[] getExistingRepos() { - return existingRepos; - } - - public void setInstalledPackages(PackageDetail[] packages) { - installedPackages = packages; - } - - public PackageDetail[] getInstalledPackages() { - return installedPackages; - } - - public void setHostHealth(HostHealth healthReport) { - hostHealth = healthReport; - } - - public HostHealth getHostHealth() { - return hostHealth; - } - - public Boolean getIptablesIsRunning() { - return iptablesIsRunning; - } - - public void setIptablesIsRunning(Boolean iptablesIsRunning) { - this.iptablesIsRunning = iptablesIsRunning; - } - - public static class HostHealth { - /** - * Java processes running on the system. Default empty array. - */ - @SerializedName("activeJavaProcs") - private JavaProc[] activeJavaProcs = new JavaProc[0]; - - /** - * The current time when agent send the host check report - */ - @SerializedName("agentTimeStampAtReporting") - private long agentTimeStampAtReporting = 0; - - /** - * The current time when host check report was received - */ - @SerializedName("serverTimeStampAtReporting") - private long serverTimeStampAtReporting = 0; - - /** - * Live services running on the agent - */ - @SerializedName("liveServices") - private LiveService[] liveServices = new LiveService[0]; - - public void setAgentTimeStampAtReporting(long currentTime) { - agentTimeStampAtReporting = currentTime; - } - - public long getAgentTimeStampAtReporting() { - return agentTimeStampAtReporting; - } - - public void setServerTimeStampAtReporting(long currentTime) { - serverTimeStampAtReporting = currentTime; - } - - public long getServerTimeStampAtReporting() { - return serverTimeStampAtReporting; - } - - public void setActiveJavaProcs(JavaProc[] procs) { - activeJavaProcs = procs; - } - - public JavaProc[] getActiveJavaProcs() { - return activeJavaProcs; - } - - public void setLiveServices(LiveService[] services) { - liveServices = services; - } - - public LiveService[] getLiveServices() { - return liveServices; - } - } - - public static class PackageDetail { - @SerializedName("name") - private String pkgName; - @SerializedName("version") - private String pkgVersion; - @SerializedName("repoName") - private String pkgRepoName; - - public void setName(String name) { - pkgName = name; - } - - public String getName() { - return pkgName; - } - - public void setVersion(String version) { - pkgVersion = version; - } - - public String getVersion() { - return pkgVersion; - } - - public void setRepoName(String repoName) { - pkgRepoName = repoName; - } - - public String getRepoName() { - return pkgRepoName; - } - } - - /** - * Represents information about a directory of interest. - */ - public static class Directory { - @SerializedName("name") - private String dirName; - @SerializedName("type") - private String dirType; - - public void setName(String name) { - dirName = name; - } - - public String getName() { - return dirName; - } - - public void setType(String type) { - dirType = type; - } - - public String getType() { - return dirType; - } - } - - /** - * Represents information about running java processes. - */ - public static class JavaProc { - @SerializedName("user") - private String user; - @SerializedName("pid") - private int pid = 0; - @SerializedName("hadoop") - private boolean is_hadoop = false; - @SerializedName("command") - private String command; - - public void setUser(String user) { - this.user = user; - } - - public String getUser() { - return user; - } - - public void setPid(int pid) { - this.pid = pid; - } - - public int getPid() { - return pid; - } - - public void setHadoop(boolean hadoop) { - is_hadoop = hadoop; - } - - public boolean isHadoop() { - return is_hadoop; - } - - public void setCommand(String cmd) { - command = cmd; - } - - public String getCommand() { - return command; - } - } - - public static class Alternative { - @SerializedName("name") - private String altName; - @SerializedName("target") - private String altTarget; - - public void setName(String name) { - altName = name; - } - - public String getName() { - return altName; - } - - public void setTarget(String target) { - altTarget = target; - } - - public String getTarget() { - return altTarget; - } - } - - public static class LiveService { - @SerializedName("name") - private String svcName; - @SerializedName("status") - private String svcStatus; - @SerializedName("desc") - private String svcDesc; - - public void setName(String name) { - svcName = name; - } - - public String getName() { - return svcName; - } - - public void setStatus(String status) { - svcStatus = status; - } - - public String getStatus() { - return svcStatus; - } - - public void setDesc(String desc) { - svcDesc = desc; - } - - public String getDesc() { - return svcDesc; - } - } - - public static class ExistingUser { - @SerializedName("name") - private String name; - @SerializedName("homeDir") - private String homeDir; - @SerializedName("status") - private String status; - - public void setUserName(String userName) { - name = userName; - } - - public String getUserName() { - return name; - } - - public void setUserHomeDir(String userHomeDir) { - homeDir = userHomeDir; - } - - public String getUserHomeDir() { - return homeDir; - } - - public void setUserStatus(String userStatus) { - status = userStatus; - } - - public String getUserStatus() { - return status; - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java deleted file mode 100644 index 20ef068c141..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -import org.apache.slider.server.appmaster.web.WebAppApi; -import org.apache.slider.server.services.security.SignCertResponse; -import org.apache.slider.server.services.security.SignMessage; -import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.Consumes; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -/** - * - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class AgentResource extends AbstractSliderResource { - - private String agent_name; - - public AgentResource(WebAppApi slider) { - super(slider); - } - - private void init(HttpServletResponse res) { - res.setContentType(null); - } - - @GET - @Path("/agent/register") - public Response endpointAgentRegister() { - Response response = Response.status(200).entity("/agent/register").build(); - return response; - } - - @GET - @Path("/agent") - public Response endpointAgent() { - Response response = Response.status(200).entity("/agent").build(); - return response; - } - @GET - @Path("/") - public Response endpointRoot() { - Response response = Response.status(200).entity("/").build(); - return response; - } - - @POST - @Path("/{agent_name: [a-zA-Z][a-zA-Z0-9_-]*}/register") - @Consumes({MediaType.APPLICATION_JSON}) - @Produces({MediaType.APPLICATION_JSON}) - public RegistrationResponse register(Register registration, - @Context HttpServletResponse res, - @PathParam("agent_name") String agent_name) { - init(res); - this.agent_name = agent_name; - AgentRestOperations ops = slider.getAgentRestOperations(); - return ops.handleRegistration(registration); - - } - - @POST - @Path("/{agent_name: [a-zA-Z][a-zA-Z0-9_-]*}/heartbeat") - @Consumes(MediaType.APPLICATION_JSON) - @Produces({MediaType.APPLICATION_JSON}) - public HeartBeatResponse heartbeat(HeartBeat message, - @Context HttpServletResponse res, - @PathParam("agent_name") String agent_name) { - init(res); - AgentRestOperations ops = slider.getAgentRestOperations(); - return ops.handleHeartBeat(message); - } - - @GET - @Path("/cert/ca") - @Produces({MediaType.TEXT_PLAIN}) - public String downloadSrvrCrt() { - return slider.getCertificateManager().getServerCert(); - } - - @Path("/certs/{hostName}") - @POST - @Consumes(MediaType.APPLICATION_JSON) - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}) - public SignCertResponse signAgentCrt(@PathParam("hostName") String hostname, - SignMessage message, @Context HttpServletRequest req) { - return slider.getCertificateManager().signAgentCrt(hostname, - message.getCsr(), - message.getPassphrase()); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentRestOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentRestOperations.java deleted file mode 100644 index 2891be86801..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentRestOperations.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -/** - * - */ -public interface AgentRestOperations { - - RegistrationResponse handleRegistration(Register registration); - - HeartBeatResponse handleHeartBeat(HeartBeat heartBeat); -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java deleted file mode 100644 index 3a3b0c0ce2c..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -import com.google.common.base.Preconditions; -import com.sun.jersey.api.core.ResourceConfig; -import com.sun.jersey.spi.container.WebApplication; -import com.sun.jersey.spi.container.servlet.ServletContainer; -import com.sun.jersey.spi.container.servlet.WebConfig; -import com.sun.jersey.spi.inject.SingletonTypeInjectableProvider; -import org.apache.slider.core.conf.MapOperations; -import org.apache.slider.providers.agent.AgentKeys; -import org.apache.slider.server.appmaster.web.WebAppApi; -import org.apache.slider.server.services.security.SecurityUtils; -import org.mortbay.jetty.Connector; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.security.SslSelectChannelConnector; -import org.mortbay.jetty.servlet.Context; -import org.mortbay.jetty.servlet.ServletHolder; -import org.mortbay.thread.QueuedThreadPool; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.ws.rs.ext.Provider; -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.net.BindException; -import java.util.Set; - -/** - * - */ -public class AgentWebApp implements Closeable { - protected static final Logger LOG = LoggerFactory.getLogger(AgentWebApp.class); - private int port; - private int securedPort; - private static Server agentServer; - public static final String BASE_PATH = "slideragent"; - - public static class Builder { - final String name; - final String wsName; - final WebAppApi application; - int port; - int securedPort; - MapOperations configsMap; - - public Builder(String name, String wsName, WebAppApi application) { - this.name = name; - this.wsName = wsName; - this.application = application; - } - - public Builder withComponentConfig(MapOperations appMasterConfig) { - this.configsMap = appMasterConfig; - return this; - } - - public Builder withPort (int port) { - this.port = port; - return this; - } - - public Builder withSecuredPort (int securedPort) { - this.securedPort = securedPort; - return this; - } - - public AgentWebApp start() throws IOException { - if (configsMap == null) { - throw new IllegalStateException("No SSL Configuration Available"); - } - - agentServer = new Server(); - agentServer.setThreadPool( - new QueuedThreadPool( - configsMap.getOptionInt("agent.threadpool.size.max", 25))); - agentServer.setStopAtShutdown(true); - agentServer.setGracefulShutdown(1000); - - SslSelectChannelConnector ssl1WayConnector = createSSLConnector(false, port); - SslSelectChannelConnector ssl2WayConnector = - createSSLConnector(Boolean.valueOf( - configsMap.getOption(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, - "false")), securedPort); - agentServer.setConnectors(new Connector[]{ssl1WayConnector, - ssl2WayConnector}); - - ServletHolder agent = new ServletHolder(new AgentServletContainer()); - Context agentRoot = new Context(agentServer, "/", Context.SESSIONS); - - agent.setInitParameter("com.sun.jersey.config.property.resourceConfigClass", - "com.sun.jersey.api.core.PackagesResourceConfig"); - agent.setInitParameter("com.sun.jersey.config.property.packages", - "org.apache.slider.server.appmaster.web.rest.agent"); - agent.setInitParameter("com.sun.jersey.api.json.POJOMappingFeature", - "true"); -// agent.setInitParameter("com.sun.jersey.spi.container.ContainerRequestFilters", "com.sun.jersey.api.container.filter.LoggingFilter"); -// agent.setInitParameter("com.sun.jersey.spi.container.ContainerResponseFilters", "com.sun.jersey.api.container.filter.LoggingFilter"); -// agent.setInitParameter("com.sun.jersey.config.feature.Trace", "true"); - agentRoot.addServlet(agent, "/*"); - - try { - openListeners(); - agentServer.start(); - } catch (IOException e) { - LOG.error("Unable to start agent server", e); - throw e; - } catch (Exception e) { - LOG.error("Unable to start agent server", e); - throw new IOException("Unable to start agent server: " + e, e); - } - - AgentWebApp webApp = new AgentWebApp(); - webApp.setPort(getConnectorPort(agentServer, 0)); - webApp.setSecuredPort(getConnectorPort(agentServer, 1)); - return webApp; - - } - - private void openListeners() throws Exception { - // from HttpServer2.openListeners() - for (Connector listener : agentServer.getConnectors()) { - if (listener.getLocalPort() != -1) { - // This listener is either started externally or has been bound - continue; - } - int port = listener.getPort(); - while (true) { - // jetty has a bug where you can't reopen a listener that previously - // failed to open w/o issuing a close first, even if the port is changed - try { - listener.close(); - listener.open(); - LOG.info("Jetty bound to port " + listener.getLocalPort()); - break; - } catch (BindException ex) { - if (port == 0) { - BindException be = new BindException("Port in use: " - + listener.getHost() + ":" + listener.getPort()); - be.initCause(ex); - throw be; - } - } - // try the next port number - listener.setPort(++port); - Thread.sleep(100); - } - } - } - - private SslSelectChannelConnector createSSLConnector(boolean needClientAuth, int port) { - SslSelectChannelConnector sslConnector = new - SslSelectChannelConnector(); - - String keystore = SecurityUtils.getSecurityDir() + - File.separator + "keystore.p12"; - String srvrCrtPass = SecurityUtils.getKeystorePass(); - sslConnector.setKeystore(keystore); - sslConnector.setTruststore(keystore); - sslConnector.setPassword(srvrCrtPass); - sslConnector.setKeyPassword(srvrCrtPass); - sslConnector.setTrustPassword(srvrCrtPass); - sslConnector.setKeystoreType("PKCS12"); - sslConnector.setTruststoreType("PKCS12"); - sslConnector.setNeedClientAuth(needClientAuth); - - sslConnector.setPort(port); - sslConnector.setAcceptors(2); - return sslConnector; - } - - @Provider - public class WebAppApiProvider extends - SingletonTypeInjectableProvider { - - public WebAppApiProvider () { - super(WebAppApi.class, application); - } - } - - public class AgentServletContainer extends ServletContainer { - public AgentServletContainer() { - super(); - } - - @Override - protected void configure(WebConfig wc, - ResourceConfig rc, - WebApplication wa) { - super.configure(wc, rc, wa); - Set singletons = rc.getSingletons(); - singletons.add(new WebAppApiProvider()); - } - } - - private int getConnectorPort(Server webServer, int index) { - Preconditions.checkArgument(index >= 0); - if (index > webServer.getConnectors().length) - throw new IllegalStateException("Illegal connect index requested"); - - Connector c = webServer.getConnectors()[index]; - if (c.getLocalPort() == -1) { - // The connector is not bounded - throw new IllegalStateException("The connector is not bound to a port"); - } - - return c.getLocalPort(); - } - } - - public static Builder $for(String name, WebAppApi app, String wsPrefix) { - return new Builder(name, wsPrefix, app); - } - - public int getPort() { - return port; - } - - public void setPort(int port) { - this.port = port; - } - - public void setSecuredPort(int securedPort) { - this.securedPort = securedPort; - } - - public int getSecuredPort() { - return securedPort; - } - - public void close() throws IOException{ - //need to stop server and reset injector - try { - agentServer.stop(); - } catch (IOException e) { - throw e; - } catch (Exception e) { - throw new IOException(e.toString(), e); - } - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebServices.java deleted file mode 100644 index 684ce6fd1fb..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebServices.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -import org.apache.slider.server.appmaster.web.WebAppApi; -import org.apache.slider.server.appmaster.web.rest.RestPaths; - -import javax.ws.rs.Path; -import javax.ws.rs.core.Context; - -/** The available agent REST services exposed by a slider AM. */ -@Path(RestPaths.SLIDER_AGENT_CONTEXT_ROOT) -public class AgentWebServices { - /** AM/WebApp info object */ - @Context - private WebAppApi slider; - - public AgentWebServices() { - } - - @Path(RestPaths.SLIDER_SUBPATH_AGENTS) - public AgentResource getAgentResource () { - return new AgentResource(slider); - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java deleted file mode 100644 index a37e4903dfe..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.Map; - -/** - * - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class CommandReport { - - int exitCode; - private String role; - private String actionId; - private String stdout; - private String stderr; - private String structuredOut; - private String status; - private String clusterName; - private String serviceName; - private long taskId; - private String roleCommand; - private Map folders; - private Map allocatedPorts; - private Map> configurationTags; - - @JsonProperty("taskId") - public long getTaskId() { - return taskId; - } - - @JsonProperty("taskId") - public void setTaskId(long taskId) { - this.taskId = taskId; - } - - @JsonProperty("clusterName") - public String getClusterName() { - return this.clusterName; - } - - @JsonProperty("clusterName") - public void setClusterName(String clusterName) { - this.clusterName = clusterName; - } - - @JsonProperty("actionId") - public String getActionId() { - return this.actionId; - } - - @JsonProperty("actionId") - public void setActionId(String actionId) { - this.actionId = actionId; - } - - @JsonProperty("stderr") - public String getStdErr() { - return this.stderr; - } - - @JsonProperty("stderr") - public void setStdErr(String stderr) { - this.stderr = stderr; - } - - @JsonProperty("exitcode") - public int getExitCode() { - return this.exitCode; - } - - @JsonProperty("exitcode") - public void setExitCode(int exitCode) { - this.exitCode = exitCode; - } - - @JsonProperty("stdout") - public String getStdOut() { - return this.stdout; - } - - @JsonProperty("stdout") - public void setStdOut(String stdout) { - this.stdout = stdout; - } - - @JsonProperty("structuredOut") - public String getStructuredOut() { - return this.structuredOut; - } - - @JsonProperty("structuredOut") - public void setStructuredOut(String structuredOut) { - this.structuredOut = structuredOut; - } - - @JsonProperty("roleCommand") - public String getRoleCommand() { - return this.roleCommand; - } - - @JsonProperty("roleCommand") - public void setRoleCommand(String roleCommand) { - this.roleCommand = roleCommand; - } - - @JsonProperty("role") - public String getRole() { - return role; - } - - @JsonProperty("role") - public void setRole(String role) { - this.role = role; - } - - @JsonProperty("status") - public String getStatus() { - return status; - } - - @JsonProperty("status") - public void setStatus(String status) { - this.status = status; - } - - @JsonProperty("serviceName") - public String getServiceName() { - return serviceName; - } - - @JsonProperty("serviceName") - public void setServiceName(String serviceName) { - this.serviceName = serviceName; - } - - /** @return the config tags that match this command, or null if none are present */ - @JsonProperty("configurationTags") - public Map> getConfigurationTags() { - return configurationTags; - } - - /** @param tags the config tags that match this command */ - @JsonProperty("configurationTags") - public void setConfigurationTags(Map> tags) { - configurationTags = tags; - } - - /** @return the allocated ports, or null if none are present */ - @JsonProperty("allocatedPorts") - public Map getAllocatedPorts() { - return allocatedPorts; - } - - /** @param ports allocated ports */ - @JsonProperty("allocatedPorts") - public void setAllocatedPorts(Map ports) { - this.allocatedPorts = ports; - } - - /** @return the folders, or null if none are present */ - @JsonProperty("folders") - public Map getFolders() { - return folders; - } - - /** @param folders allocated ports */ - @JsonProperty("folders") - public void setFolders(Map folders) { - this.folders = folders; - } - - @Override - public String toString() { - return "CommandReport{" + - "role='" + role + '\'' + - ", actionId='" + actionId + '\'' + - ", status='" + status + '\'' + - ", exitCode=" + exitCode + - ", clusterName='" + clusterName + '\'' + - ", serviceName='" + serviceName + '\'' + - ", taskId=" + taskId + - ", roleCommand=" + roleCommand + - ", configurationTags=" + configurationTags + - '}'; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java deleted file mode 100644 index acdc234694d..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.Map; - -/** - * - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class ComponentStatus { - String componentName; - String msg; - String status; - String serviceName; - String clusterName; - String roleCommand; - String ip; - String hostname; - @JsonProperty("configurations") - private Map> configurations; - - public String getRoleCommand() { - return roleCommand; - } - - public void setRoleCommand(String roleCommand) { - this.roleCommand = roleCommand; - } - - public String getComponentName() { - return this.componentName; - } - - public void setComponentName(String componentName) { - this.componentName = componentName; - } - - public String getMessage() { - return this.msg; - } - - public void setMessage(String msg) { - this.msg = msg; - } - - public String getStatus() { - return this.status; - } - - public void setStatus(String status) { - this.status = status; - } - - public String getServiceName() { - return serviceName; - } - - public void setServiceName(String serviceName) { - this.serviceName = serviceName; - } - - public String getClusterName() { - return clusterName; - } - - public void setClusterName(String clusterName) { - this.clusterName = clusterName; - } - - /** @return the config tags that match this command, or null if none are present */ - public Map> getConfigs() { - return configurations; - } - - /** @param configs the config tags that match this status */ - public void setConfigs(Map> configs) { - this.configurations = configs; - } - - @Override - public String toString() { - return "ComponentStatus{" + - "componentName='" + componentName + '\'' + - ", msg='" + msg + '\'' + - ", status='" + status + '\'' + - ", serviceName='" + serviceName + '\'' + - ", clusterName='" + clusterName + '\'' + - ", roleCommand='" + roleCommand + '\'' + - ", ip='" + ip + '\'' + - ", hostname='" + hostname + '\'' + - '}'; - } - - public String getIp() { - return ip; - } - - public void setIp(String ip) { - this.ip = ip; - } - - public String getHostname() { - return hostname; - } - - public void setHostname(String hostname) { - this.hostname = hostname; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/DiskInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/DiskInfo.java deleted file mode 100644 index 27c4d54ce60..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/DiskInfo.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -/** - * - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class DiskInfo { - String available; - String mountpoint; - String device; - String used; - String percent; - String size; - String type; - - /** - * DiskInfo object that tracks information about a disk. - * @param mountpoint - * @param available - * @param used - * @param percent - * @param size - */ - public DiskInfo(String device, String mountpoint, String available, - String used, String percent, String size, String type) { - this.device = device; - this.mountpoint = mountpoint; - this.available = available; - this.used = used; - this.percent = percent; - this.size = size; - this.type = type; - } - - /** - * Needed for Serialization - */ - public DiskInfo() {} - - @JsonProperty("available") - public void setAvailable(String available) { - this.available = available; - } - - @JsonProperty("available") - public String getAvailable() { - return this.available; - } - - @JsonProperty("mountpoint") - public String getMountPoint() { - return this.mountpoint; - } - - @JsonProperty("mountpoint") - public void setMountPoint(String mountpoint) { - this.mountpoint = mountpoint; - } - - @JsonProperty("type") - public String getType() { - return this.type; - } - - @JsonProperty("type") - public void setType(String type) { - this.type = type; - } - - @JsonProperty("used") - public String getUsed() { - return this.used; - } - - @JsonProperty("used") - public void setUsed(String used) { - this.used = used; - } - - @JsonProperty("percent") - public String getPercent() { - return this.percent; - } - - @JsonProperty("percent") - public void setPercent(String percent) { - this.percent = percent; - } - - @JsonProperty("size") - public String getSize() { - return this.size; - } - - @JsonProperty("size") - public void setSize(String size) { - this.size = size; - } - - @Override - public String toString() { - return "available=" + this.available + ",mountpoint=" + this.mountpoint - + ",used=" + this.used + ",percent=" + this.percent + ",size=" + - this.size + ",device=" + this.device + - ",type=" + this.type; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java deleted file mode 100644 index d3864b85a13..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java +++ /dev/null @@ -1,310 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -import org.apache.slider.providers.agent.application.metadata.Component; -import org.apache.slider.providers.agent.application.metadata.DockerContainer; -import org.apache.slider.providers.agent.application.metadata.DockerContainerInputFile; -import org.apache.slider.providers.agent.application.metadata.DockerContainerMount; -import org.apache.slider.providers.agent.application.metadata.DockerContainerPort; -import org.apache.slider.providers.agent.application.metadata.Metainfo; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class ExecutionCommand { - protected static final Logger log = - LoggerFactory.getLogger(ExecutionCommand.class); - private AgentCommandType commandType = AgentCommandType.EXECUTION_COMMAND; - private String clusterName; - private long taskId; - private String commandId; - //TODO Remove hostname from being set in the command - private String hostname; - private String role; - private Map hostLevelParams = new HashMap(); - private Map roleParams = null; - private String roleCommand; - private Map> configurations; - private Map> componentConfigurations; - private Map commandParams; - private String serviceName; - private String componentName; - private String componentType; - private List containers = new ArrayList<>(); - private String pkg; - private boolean yarnDockerMode = false; - - public ExecutionCommand(AgentCommandType commandType) { - this.commandType = commandType; - } - - @JsonProperty("commandType") - public AgentCommandType getCommandType() { - return commandType; - } - - @JsonProperty("commandType") - public void setCommandType(AgentCommandType commandType) { - this.commandType = commandType; - } - - @JsonProperty("commandId") - public String getCommandId() { - return this.commandId; - } - - @JsonProperty("commandId") - public void setCommandId(String commandId) { - this.commandId = commandId; - } - - @JsonProperty("taskId") - public long getTaskId() { - return taskId; - } - - @JsonProperty("taskId") - public void setTaskId(long taskId) { - this.taskId = taskId; - } - - @JsonProperty("role") - public String getRole() { - return role; - } - - @JsonProperty("role") - public void setRole(String role) { - this.role = role; - } - - @JsonProperty("roleParams") - public Map getRoleParams() { - return roleParams; - } - - @JsonProperty("roleParams") - public void setRoleParams(Map roleParams) { - this.roleParams = roleParams; - } - - @JsonProperty("roleCommand") - public String getRoleCommand() { - return roleCommand; - } - - @JsonProperty("roleCommand") - public void setRoleCommand(String cmd) { - this.roleCommand = cmd; - } - - @JsonProperty("clusterName") - public String getClusterName() { - return clusterName; - } - - @JsonProperty("clusterName") - public void setClusterName(String clusterName) { - this.clusterName = clusterName; - } - - @JsonProperty("componentType") - public String getComponentType() { - return componentType; - } - - @JsonProperty("componentType") - public void setComponentType(String componentType) { - this.componentType = componentType; - } - - @JsonProperty("hostname") - public String getHostname() { - return hostname; - } - - @JsonProperty("hostname") - public void setHostname(String hostname) { - this.hostname = hostname; - } - - @JsonProperty("hostLevelParams") - public Map getHostLevelParams() { - return hostLevelParams; - } - - @JsonProperty("hostLevelParams") - public void setHostLevelParams(Map params) { - this.hostLevelParams = params; - } - - @JsonProperty("configurations") - public Map> getConfigurations() { - return configurations; - } - - @JsonProperty("configurations") - public void setConfigurations(Map> configurations) { - this.configurations = configurations; - } - - @JsonProperty("commandParams") - public Map getCommandParams() { - return commandParams; - } - - @JsonProperty("commandParams") - public void setCommandParams(Map commandParams) { - this.commandParams = commandParams; - } - - @JsonProperty("serviceName") - public String getServiceName() { - return serviceName; - } - - @JsonProperty("serviceName") - public void setServiceName(String serviceName) { - this.serviceName = serviceName; - } - - @JsonProperty("componentName") - public String getComponentName() { - return componentName; - } - - @JsonProperty("componentName") - public void setComponentName(String componentName) { - this.componentName = componentName; - } - - @JsonProperty("package") - public String getPkg() { - return pkg; - } - - @JsonProperty("package") - public void setPkg(String pkg) { - this.pkg = pkg; - } - - @JsonProperty("componentConfig") - public Map> getComponentConfigurations() { - return componentConfigurations; - } - - @JsonProperty("componentConfig") - public void setComponentConfigurations( - Map> componentConfigurations) { - this.componentConfigurations = componentConfigurations; - } - - @JsonProperty("containers") - public List getContainers() { - return containers; - } - - @JsonProperty("yarnDockerMode") - public boolean isYarnDockerMode() { - return yarnDockerMode ; - } - - @JsonProperty("yarnDockerMode") - public void setYarnDockerMode(boolean yarnDockerMode) { - this.yarnDockerMode = yarnDockerMode; - } - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("ExecutionCommand [commandType=").append(commandType) - .append(", clusterName=").append(clusterName).append(", taskId=") - .append(taskId).append(", commandId=").append(commandId) - .append(", hostname=").append(hostname).append(", role=").append(role) - .append(", hostLevelParams=").append(hostLevelParams) - .append(", roleParams=").append(roleParams).append(", roleCommand=") - .append(roleCommand).append(", configurations=").append(configurations) - .append(", commandParams=").append(commandParams) - .append(", serviceName=").append(serviceName) - .append(", componentName=").append(componentName) - .append(", componentType=").append(componentType) - .append(", yarnDockerMode=").append(yarnDockerMode).append(", pkg=") - .append(pkg).append("]"); - return builder.toString(); - } - - public void addContainerDetails(String componentGroup, Metainfo metaInfo) { - Component component = metaInfo.getApplicationComponent(componentGroup); - this.setComponentType(component.getType()); - log.info("Adding container details for {}", componentGroup, " from ", - metaInfo.toString()); - for (DockerContainer metaContainer : component.getDockerContainers()) { - DockerContainer container = new DockerContainer(); - container.setImage(metaContainer.getImage()); - container.setNetwork(metaContainer.getNetwork()); - container.setUseNetworkScript(metaContainer.getUseNetworkScript()); - container.setName(metaContainer.getName()); - container.setOptions(metaContainer.getOptions()); - container.setAdditionalParam(metaContainer.getAdditionalParam()); - container.setCommandPath(metaContainer.getAdditionalParam()); - container.setStatusCommand(metaContainer.getStatusCommand()); - container.setStartCommand(metaContainer.getStartCommand()); - if (metaContainer.getMounts().size() > 0) { - for (DockerContainerMount metaContMount : metaContainer.getMounts()) { - DockerContainerMount contMnt = new DockerContainerMount(); - contMnt.setContainerMount(metaContMount.getContainerMount()); - contMnt.setHostMount(metaContMount.getHostMount()); - container.getMounts().add(contMnt); - } - } - if (metaContainer.getPorts().size() > 0) { - for (DockerContainerPort metaCntPort : metaContainer.getPorts()) { - DockerContainerPort cntPort = new DockerContainerPort(); - cntPort.setContainerPort(metaCntPort.getContainerPort()); - cntPort.setHostPort(metaCntPort.getHostPort()); - container.getPorts().add(cntPort); - } - } - if (metaContainer.getInputFiles().size() > 0) { - for (DockerContainerInputFile metaInpFile : metaContainer - .getInputFiles()) { - DockerContainerInputFile inpFile = new DockerContainerInputFile(); - inpFile.setContainerMount(metaInpFile.getContainerMount()); - inpFile.setFileLocalPath(metaInpFile.getFileLocalPath()); - container.getInputFiles().add(inpFile); - } - } - if (metaContainer.getConfigFiles() != null) { - container.setConfigFiles(metaContainer.getConfigFiles()); - } - log.info("Docker container meta info ready: " + container.toString()); - this.getContainers().add(container); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java deleted file mode 100644 index d17c465d65d..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.web.rest.agent; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.ArrayList; -import java.util.List; - -/** - * - * - * Data model for agent heartbeat for server (ambari or app master). - * - */ - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class HeartBeat { - private long responseId = -1; - private long timestamp; - private String hostname; - List reports = new ArrayList(); - List componentStatus = new ArrayList(); - private List mounts = new ArrayList(); - HostStatus nodeStatus; - private AgentEnv agentEnv = null; - private String fqdn; - private String pkg; - - public long getResponseId() { - return responseId; - } - - public void setResponseId(long responseId) { - this.responseId=responseId; - } - - public long getTimestamp() { - return timestamp; - } - - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; - } - - public String getHostname() { - return hostname; - } - - public void setHostname(String hostname) { - this.hostname = hostname; - } - - public String getFqdn() { - return fqdn; - } - - public void setFqdn(String fqdn) { - this.fqdn = fqdn; - } - - @JsonProperty("reports") - public List getReports() { - return this.reports; - } - - @JsonProperty("reports") - public void setReports(List reports) { - this.reports = reports; - } - - public HostStatus getNodeStatus() { - return nodeStatus; - } - - public void setNodeStatus(HostStatus nodeStatus) { - this.nodeStatus = nodeStatus; - } - - public AgentEnv getAgentEnv() { - return agentEnv; - } - - public void setAgentEnv(AgentEnv env) { - agentEnv = env; - } - - @JsonProperty("componentStatus") - public List getComponentStatus() { - return componentStatus; - } - - @JsonProperty("componentStatus") - public void setComponentStatus(List componentStatus) { - this.componentStatus = componentStatus; - } - - @JsonProperty("mounts") - public List getMounts() { - return this.mounts; - } - - @JsonProperty("mounts") - public void setMounts(List mounts) { - this.mounts = mounts; - } - - @JsonProperty("package") - public String getPackage() { - return pkg; - } - - @JsonProperty("package") - public void setPackage(String pkg) { - this.pkg = pkg; - } - - @Override - public String toString() { - return "HeartBeat{" + - "responseId=" + responseId + - ", timestamp=" + timestamp + - ", hostname='" + hostname + '\'' + - ", reports=" + reports + - ", componentStatus=" + componentStatus + - ", package=" + pkg + - ", nodeStatus=" + nodeStatus + - '}'; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java deleted file mode 100644 index b500d67ab2d..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.ArrayList; -import java.util.List; - -/** - * - * Controller to Agent response data model. - * - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class HeartBeatResponse { - - private long responseId; - - List executionCommands = new ArrayList(); - List statusCommands = new ArrayList(); - - RegistrationCommand registrationCommand; - - boolean yarnDockerMode = false; - boolean restartAgent = false; - boolean restartEnabled = true; - boolean hasMappedComponents = false; - boolean terminateAgent = false; - - @JsonProperty("responseId") - public long getResponseId() { - return responseId; - } - - @JsonProperty("responseId") - public void setResponseId(long responseId) { - this.responseId=responseId; - } - - @JsonProperty("executionCommands") - public List getExecutionCommands() { - return executionCommands; - } - - @JsonProperty("executionCommands") - public void setExecutionCommands(List executionCommands) { - this.executionCommands = executionCommands; - } - - @JsonProperty("statusCommands") - public List getStatusCommands() { - return statusCommands; - } - - @JsonProperty("statusCommands") - public void setStatusCommands(List statusCommands) { - this.statusCommands = statusCommands; - } - - @JsonProperty("registrationCommand") - public RegistrationCommand getRegistrationCommand() { - return registrationCommand; - } - - @JsonProperty("registrationCommand") - public void setRegistrationCommand(RegistrationCommand registrationCommand) { - this.registrationCommand = registrationCommand; - } - - @JsonProperty("restartAgent") - public boolean isRestartAgent() { - return restartAgent; - } - - @JsonProperty("restartAgent") - public void setRestartAgent(boolean restartAgent) { - this.restartAgent = restartAgent; - } - - @JsonProperty("restartEnabled") - public boolean getRstartEnabled() { - return restartEnabled; - } - - @JsonProperty("restartEnabled") - public void setRestartEnabled(boolean restartEnabled) { - this.restartEnabled = restartEnabled; - } - - @JsonProperty("hasMappedComponents") - public boolean hasMappedComponents() { - return hasMappedComponents; - } - - @JsonProperty("hasMappedComponents") - public void setHasMappedComponents(boolean hasMappedComponents) { - this.hasMappedComponents = hasMappedComponents; - } - - @JsonProperty("terminateAgent") - public boolean isTerminateAgent() { - return terminateAgent; - } - - @JsonProperty("terminateAgent") - public void setTerminateAgent(boolean terminateAgent) { - this.terminateAgent = terminateAgent; - } - - public void addExecutionCommand(ExecutionCommand execCmd) { - executionCommands.add(execCmd); - } - - public void addStatusCommand(StatusCommand statCmd) { - statusCommands.add(statCmd); - } - - @Override - public String toString() { - return "HeartBeatResponse{" + - "responseId=" + responseId + - ", executionCommands=" + executionCommands + - ", statusCommands=" + statusCommands + - ", registrationCommand=" + registrationCommand + - ", restartAgent=" + restartAgent + - ", terminateAgent=" + terminateAgent + - '}'; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostInfo.java deleted file mode 100644 index bef7b0719a2..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostInfo.java +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.ArrayList; -import java.util.List; - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class HostInfo { - private String architecture; - private String domain; - private String fqdn; - private String hardwareisa; - private String hardwaremodel; - private String hostname; - private String id; - private String interfaces; - private String ipaddress; - private String kernel; - private String kernelmajversion; - private String kernelrelease; - private String kernelversion; - private String macaddress; - private long memoryfree; - private long memorysize; - private List mounts = new ArrayList(); - private long memorytotal; - private String netmask; - private String operatingsystem; - private String operatingsystemrelease; - private String osfamily; - private int physicalprocessorcount; - private int processorcount; - private boolean selinux; - private String swapfree; - private String swapsize; - private String timezone; - private String uptime; - private long uptime_days; - private long uptime_hours; - - - @JsonProperty("architecture") - public String getArchitecture() { - return this.architecture; - } - - @JsonProperty("architecture") - public void setArchitecture(String architecture) { - this.architecture = architecture; - } - - @JsonProperty("domain") - public String getDomain() { - return this.domain; - } - - @JsonProperty("domain") - public void setDomain(String domain) { - this.domain = domain; - } - - @JsonProperty("fqdn") - public String getFQDN() { - return this.fqdn; - } - - @JsonProperty("fqdn") - public void setFQDN(String fqdn) { - this.fqdn = fqdn; - } - - @JsonProperty("hardwareisa") - public String getHardwareIsa() { - return hardwareisa; - } - - @JsonProperty("hardwareisa") - public void setHardwareIsa(String hardwareisa) { - this.hardwareisa = hardwareisa; - } - - @JsonProperty("hardwaremodel") - public String getHardwareModel() { - return this.hardwaremodel; - } - - @JsonProperty("hardwaremodel") - public void setHardwareModel(String hardwaremodel) { - this.hardwaremodel = hardwaremodel; - } - - @JsonProperty("hostname") - public String getHostName() { - return this.hostname; - } - - @JsonProperty("hostname") - public void setHostName(String hostname) { - this.hostname = hostname; - } - - @JsonProperty("id") - public String getAgentUserId() { - return id; - } - - @JsonProperty("id") - public void setAgentUserId(String id) { - this.id = id; - } - - @JsonProperty("interfaces") - public String getInterfaces() { - return this.interfaces; - } - - @JsonProperty("interfaces") - public void setInterfaces(String interfaces) { - this.interfaces = interfaces; - } - - @JsonProperty("ipaddress") - public String getIPAddress() { - return this.ipaddress; - } - - @JsonProperty("ipaddress") - public void setIPAddress(String ipaddress) { - this.ipaddress = ipaddress; - } - - @JsonProperty("kernel") - public String getKernel() { - return this.kernel; - } - - @JsonProperty("kernel") - public void setKernel(String kernel) { - this.kernel = kernel; - } - - @JsonProperty("kernelmajversion") - public String getKernelMajVersion() { - return this.kernelmajversion; - } - - @JsonProperty("kernelmajversion") - public void setKernelMajVersion(String kernelmajversion) { - this.kernelmajversion = kernelmajversion; - } - - @JsonProperty("kernelrelease") - public String getKernelRelease() { - return this.kernelrelease; - } - - @JsonProperty("kernelrelease") - public void setKernelRelease(String kernelrelease) { - this.kernelrelease = kernelrelease; - } - - @JsonProperty("kernelversion") - public String getKernelVersion() { - return this.kernelversion; - } - - @JsonProperty("kernelversion") - public void setKernelVersion(String kernelversion) { - this.kernelversion = kernelversion; - } - - @JsonProperty("macaddress") - public String getMacAddress() { - return this.macaddress; - } - - @JsonProperty("macaddress") - public void setMacAddress(String macaddress) { - this.macaddress = macaddress; - } - - @JsonProperty("memoryfree") - public long getFreeMemory() { - return this.memoryfree; - } - - @JsonProperty("memoryfree") - public void setFreeMemory(long memoryfree) { - this.memoryfree = memoryfree; - } - - @JsonProperty("memorysize") - public long getMemorySize() { - return this.memorysize; - } - - @JsonProperty("memorysize") - public void setMemorySize(long memorysize) { - this.memorysize = memorysize; - } - - @JsonProperty("mounts") - public List getMounts() { - return this.mounts; - } - - @JsonProperty("mounts") - public void setMounts(List mounts) { - this.mounts = mounts; - } - - @JsonProperty("memorytotal") - public long getMemoryTotal() { - return this.memorytotal; - } - - @JsonProperty("memorytotal") - public void setMemoryTotal(long memorytotal) { - this.memorytotal = memorytotal; - } - - @JsonProperty("netmask") - public String getNetMask() { - return this.netmask; - } - - @JsonProperty("netmask") - public void setNetMask(String netmask) { - this.netmask = netmask; - } - - @JsonProperty("operatingsystem") - public String getOS() { - return this.operatingsystem; - } - - @JsonProperty("operatingsystem") - public void setOS(String operatingsystem) { - this.operatingsystem = operatingsystem; - } - - @JsonProperty("operatingsystemrelease") - public String getOSRelease() { - return this.operatingsystemrelease; - } - - @JsonProperty("operatingsystemrelease") - public void setOSRelease(String operatingsystemrelease) { - this.operatingsystemrelease = operatingsystemrelease; - } - - @JsonProperty("osfamily") - public String getOSFamily() { - return this.osfamily; - } - - @JsonProperty("osfamily") - public void setOSFamily(String osfamily) { - this.osfamily = osfamily; - } - - @JsonProperty("physicalprocessorcount") - public int getPhysicalProcessorCount() { - return this.physicalprocessorcount; - } - - @JsonProperty("physicalprocessorcount") - public void setPhysicalProcessorCount(int physicalprocessorcount) { - this.physicalprocessorcount = physicalprocessorcount; - } - - @JsonProperty("processorcount") - public int getProcessorCount() { - return this.processorcount; - } - - @JsonProperty("processorcount") - public void setProcessorCount(int processorcount) { - this.processorcount = processorcount; - } - - @JsonProperty("selinux") - public boolean getSeLinux() { - return selinux; - } - - @JsonProperty("selinux") - public void setSeLinux(boolean selinux) { - this.selinux = selinux; - } - - @JsonProperty("swapfree") - public String getSwapFree() { - return this.swapfree; - } - - @JsonProperty("swapfree") - public void setSwapFree(String swapfree) { - this.swapfree = swapfree; - } - - @JsonProperty("swapsize") - public String getSwapSize() { - return swapsize; - } - - @JsonProperty("swapsize") - public void setSwapSize(String swapsize) { - this.swapsize = swapsize; - } - - @JsonProperty("timezone") - public String getTimeZone() { - return this.timezone; - } - - @JsonProperty("timezone") - public void setTimeZone(String timezone) { - this.timezone = timezone; - } - - @JsonProperty("uptime") - public String getUptime() { - return this.uptime; - } - - @JsonProperty("uptime") - public void setUpTime(String uptime) { - this.uptime = uptime; - } - - @JsonProperty("uptime_hours") - public long getUptimeHours() { - return this.uptime_hours; - } - - @JsonProperty("uptime_hours") - public void setUpTimeHours(long uptime_hours) { - this.uptime_hours = uptime_hours; - } - - @JsonProperty("uptime_days") - public long getUpTimeDays() { - return this.uptime_days; - } - - @JsonProperty("uptime_days") - public void setUpTimeDays(long uptime_days) { - this.uptime_days = uptime_days; - } - - private String getDiskString() { - if (mounts == null) { - return null; - } - StringBuilder ret = new StringBuilder(); - for (DiskInfo diskInfo : mounts) { - ret.append("(").append(diskInfo.toString()).append(")"); - } - return ret.toString(); - } - - public String toString() { - return "[" + - "hostname=" + this.hostname + "," + - "fqdn=" + this.fqdn + "," + - "domain=" + this.domain + "," + - "architecture=" + this.architecture + "," + - "processorcount=" + this.processorcount + "," + - "physicalprocessorcount=" + this.physicalprocessorcount + "," + - "osname=" + this.operatingsystem + "," + - "osversion=" + this.operatingsystemrelease + "," + - "osfamily=" + this.osfamily + "," + - "memory=" + this.memorytotal + "," + - "uptime_hours=" + this.uptime_hours + "," + - "mounts=" + getDiskString() + "]\n"; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostStatus.java deleted file mode 100644 index c584149841d..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostStatus.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -/** - * - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class HostStatus { - public HostStatus(Status status, String cause) { - super(); - this.status = status; - this.cause = cause; - } - public HostStatus() { - super(); - } - - public enum Status { - HEALTHY, - UNHEALTHY - } - Status status; - String cause; - public Status getStatus() { - return status; - } - public void setStatus(Status status) { - this.status = status; - } - public String getCause() { - return cause; - } - public void setCause(String cause) { - this.cause = cause; - } - - @Override - public String toString() { - return "HostStatus{" + - "status=" + status + - ", cause='" + cause + '\'' + - '}'; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java deleted file mode 100644 index 01500793f5e..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -import org.apache.slider.providers.agent.State; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.Map; - -/** Data model for agent to send heartbeat to ambari and/or app master. */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class Register { - private int responseId = -1; - private long timestamp; - private String label; - private int currentPingPort; - private HostInfo hardwareProfile; - private String publicHostname; - private String tags; - private AgentEnv agentEnv; - private String agentVersion; - private State actualState; - private State expectedState; - private Map allocatedPorts; - private Map logFolders; - private String pkg; - private String appVersion; - - @JsonProperty("responseId") - public int getResponseId() { - return responseId; - } - - @JsonProperty("responseId") - public void setResponseId(int responseId) { - this.responseId = responseId; - } - - public long getTimestamp() { - return timestamp; - } - - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; - } - - public String getLabel() { - return label; - } - - public void setLabel(String label) { - this.label = label; - } - - public String getTags() { - return tags; - } - - public void setTags(String tags) { - this.tags = tags; - } - - public HostInfo getHardwareProfile() { - return hardwareProfile; - } - - public void setHardwareProfile(HostInfo hardwareProfile) { - this.hardwareProfile = hardwareProfile; - } - - public String getPublicHostname() { - return publicHostname; - } - - public void setPublicHostname(String name) { - this.publicHostname = name; - } - - public AgentEnv getAgentEnv() { - return agentEnv; - } - - public void setAgentEnv(AgentEnv env) { - this.agentEnv = env; - } - - public String getAgentVersion() { - return agentVersion; - } - - public void setAgentVersion(String agentVersion) { - this.agentVersion = agentVersion; - } - - public int getCurrentPingPort() { - return currentPingPort; - } - - public void setCurrentPingPort(int currentPingPort) { - this.currentPingPort = currentPingPort; - } - - public State getActualState() { - return actualState; - } - - public void setActualState(State actualState) { - this.actualState = actualState; - } - - public State getExpectedState() { - return expectedState; - } - - public void setExpectedState(State expectedState) { - this.expectedState = expectedState; - } - - /** @return the allocated ports, or null if none are present */ - @JsonProperty("allocatedPorts") - public Map getAllocatedPorts() { - return allocatedPorts; - } - - /** @param ports allocated ports */ - @JsonProperty("allocatedPorts") - public void setAllocatedPorts(Map ports) { - this.allocatedPorts = ports; - } - - /** @return the log folders, or null if none are present */ - @JsonProperty("logFolders") - public Map getLogFolders() { - return logFolders; - } - - /** @param logFolders assigned log folders */ - @JsonProperty("logFolders") - public void setLogFolders(Map logFolders) { - this.logFolders = logFolders; - } - - public String getPkg() { - return pkg; - } - - public void setPkg(String pkg) { - this.pkg = pkg; - } - - @JsonProperty("appVersion") - public String getAppVersion() { - return appVersion; - } - - @JsonProperty("appVersion") - public void setAppVersion(String appVersion) { - this.appVersion = appVersion; - } - - @Override - public String toString() { - String ret = "responseId=" + responseId + "\n" + - "timestamp=" + timestamp + "\n" + - "label=" + label + "\n" + - "hostname=" + publicHostname + "\n" + - "expectedState=" + expectedState + "\n" + - "actualState=" + actualState + "\n" + - "appVersion=" + appVersion + "\n"; - - if (hardwareProfile != null) { - ret = ret + "hardwareprofile=" + this.hardwareProfile.toString(); - } - return ret; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationCommand.java deleted file mode 100644 index 4b87dd27d09..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationCommand.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -/** - * - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class RegistrationCommand { - - private String command; - - public String getCommand() { - return command; - } - - public void setCommand(String command) { - this.command = command; - } - - public RegistrationCommand(String command) { - - this.command = command; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java deleted file mode 100644 index 80b7a5ef05f..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.List; - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class RegistrationResponse { - - @JsonProperty("response") - private RegistrationStatus response; - - /** - * exitstatus is a code of error which was rised on server side. exitstatus - * = 0 (OK - Default) exitstatus = 1 (Registration failed because different - * version of agent and server) - */ - @JsonProperty("exitstatus") - private int exitstatus; - - /** log - message, which will be printed to agents log */ - @JsonProperty("log") - private String log; - - /** tags - tags associated with the container */ - @JsonProperty("tags") - private String tags; - - @JsonProperty("package") - private String pkg; - - //Response id to start with, usually zero. - @JsonProperty("responseId") - private long responseId; - - @JsonProperty("statusCommands") - private List statusCommands = null; - - public RegistrationResponse() { - } - - public RegistrationStatus getResponse() { - return response; - } - - public void setResponse(RegistrationStatus response) { - this.response = response; - } - - public int getExitstatus() { - return exitstatus; - } - - public void setExitstatus(int exitstatus) { - this.exitstatus = exitstatus; - } - - public RegistrationStatus getResponseStatus() { - return response; - } - - public void setResponseStatus(RegistrationStatus response) { - this.response = response; - } - - public List getStatusCommands() { - return statusCommands; - } - - public void setStatusCommands(List statusCommands) { - this.statusCommands = statusCommands; - } - - public long getResponseId() { - return responseId; - } - - public void setResponseId(long responseId) { - this.responseId = responseId; - } - - public String getTags() { - return tags; - } - - public void setTags(String tags) { - this.tags = tags; - } - - public String getLog() { - return log; - } - - public void setLog(String log) { - this.log = log; - } - - public String getPkg() { - return pkg; - } - - public void setPkg(String pkg) { - this.pkg = pkg; - } - - @Override - public String toString() { - return "RegistrationResponse{" + - "response=" + response + - ", responseId=" + responseId + - ", statusCommands=" + statusCommands + - '}'; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java deleted file mode 100644 index 83747100da6..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -public enum RegistrationStatus { - OK, - FAILED -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java deleted file mode 100644 index 5b205b59afa..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.agent; - -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.util.HashMap; -import java.util.Map; - -/** - * Command to report the status of a list of services in roles. - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -public class StatusCommand { - public static String STATUS_COMMAND = "STATUS"; - public static String GET_CONFIG_COMMAND = "GET_CONFIG"; - - AgentCommandType agentCommandType; - - private String clusterName; - private String serviceName; - private String componentName; - private Map> configurations; - private Map commandParams = new HashMap(); - private Map hostLevelParams = new HashMap(); - private String roleCommand; - private boolean yarnDockerMode; - - public StatusCommand() { - this.agentCommandType = AgentCommandType.STATUS_COMMAND; - } - - @JsonProperty("clusterName") - public String getClusterName() { - return clusterName; - } - - @JsonProperty("clusterName") - public void setClusterName(String clusterName) { - this.clusterName = clusterName; - } - - @JsonProperty("serviceName") - public String getServiceName() { - return serviceName; - } - - @JsonProperty("serviceName") - public void setServiceName(String serviceName) { - this.serviceName = serviceName; - } - - @JsonProperty("componentName") - public String getComponentName() { - return componentName; - } - - @JsonProperty("componentName") - public void setComponentName(String componentName) { - this.componentName = componentName; - } - - @JsonProperty("configurations") - public Map> getConfigurations() { - return configurations; - } - - @JsonProperty("configurations") - public void setConfigurations(Map> configurations) { - this.configurations = configurations; - } - - @JsonProperty("hostLevelParams") - public Map getHostLevelParams() { - return hostLevelParams; - } - - @JsonProperty("hostLevelParams") - public void setHostLevelParams(Map params) { - this.hostLevelParams = params; - } - - @JsonProperty("commandParams") - public Map getCommandParams() { - return commandParams; - } - - @JsonProperty("commandParams") - public void setCommandParams(Map commandParams) { - this.commandParams = commandParams; - } - - @JsonProperty("commandType") - public AgentCommandType getCommandType() { - return agentCommandType; - } - - @JsonProperty("commandType") - public void setCommandType(AgentCommandType commandType) { - this.agentCommandType = commandType; - } - - @JsonProperty("roleCommand") - public String getRoleCommand() { - return roleCommand; - } - - @JsonProperty("roleCommand") - public void setRoleCommand(String roleCommand) { - this.roleCommand = roleCommand; - } - - @JsonProperty("yarnDockerMode") - public boolean isYarnDockerMode() { - return yarnDockerMode; - } - - @JsonProperty("yarnDockerMode") - public void setYarnDockerMode(boolean yarnDockerMode) { - this.yarnDockerMode = yarnDockerMode; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("StatusCommand [agentCommandType=").append(agentCommandType) - .append(", clusterName=").append(clusterName).append(", serviceName=") - .append(serviceName).append(", componentName=").append(componentName) - .append(", configurations=").append(configurations) - .append(", commandParams=").append(commandParams) - .append(", hostLevelParams=").append(hostLevelParams) - .append(", roleCommand=").append(roleCommand).append("]"); - return builder.toString(); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java deleted file mode 100644 index 0bea8fa7e65..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.core.conf.AggregateConf; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.tools.TestUtility; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Set; - -/** - * - */ -public class TestAgentClientProvider { - protected static final Logger log = - LoggerFactory.getLogger(TestAgentClientProvider.class); - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Test - public void testGetApplicationTags() throws Exception { - Configuration configuration = new Configuration(); - FileSystem fs = FileSystem.getLocal(configuration); - SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); - - AgentClientProvider provider = new AgentClientProvider(null); - String zipFileName = TestUtility.createAppPackage( - folder, - "testpkg", - "test.zip", - "target/test-classes/org/apache/slider/common/tools/test"); - Set tags = provider.getApplicationTags(sliderFileSystem, zipFileName); - assert tags != null; - assert !tags.isEmpty(); - assert tags.contains("Name: STORM"); - assert tags.contains("Description: Apache Hadoop Stream processing framework"); - assert tags.contains("Version: 0.9.1.2.1"); - - } - - @Test - public void testValidateInstanceDefinition() throws Exception { - AgentClientProvider provider = new AgentClientProvider(null); - AggregateConf instanceDefinition = new AggregateConf(); - - try { - provider.validateInstanceDefinition(instanceDefinition, null); - Assert.assertFalse("Should fail with BadConfigException", true); - } catch (BadConfigException e) { - log.info(e.toString()); - Assert.assertTrue(e.getMessage().contains("Application definition must be provided")); - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java deleted file mode 100644 index ec62b54f57d..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent; - -import org.junit.Assert; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -/** - * - */ -public class TestAgentLaunchParameter { - protected static final Logger log = - LoggerFactory.getLogger(TestAgentLaunchParameter.class); - - @Test - public void testTestAgentLaunchParameter() throws Exception { - AgentLaunchParameter alp = new AgentLaunchParameter(""); - Assert.assertEquals("", alp.getNextLaunchParameter("abc")); - Assert.assertEquals("", alp.getNextLaunchParameter("HBASE_MASTER")); - - alp = new AgentLaunchParameter("a:1:2:3|b:5:6:NONE"); - Assert.assertEquals("1", alp.getNextLaunchParameter("a")); - Assert.assertEquals("2", alp.getNextLaunchParameter("a")); - Assert.assertEquals("3", alp.getNextLaunchParameter("a")); - Assert.assertEquals("3", alp.getNextLaunchParameter("a")); - - Assert.assertEquals("5", alp.getNextLaunchParameter("b")); - Assert.assertEquals("6", alp.getNextLaunchParameter("b")); - Assert.assertEquals("", alp.getNextLaunchParameter("b")); - Assert.assertEquals("", alp.getNextLaunchParameter("b")); - Assert.assertEquals("", alp.getNextLaunchParameter("c")); - - alp = new AgentLaunchParameter("|a:1:3|b::5:NONE:"); - Assert.assertEquals("1", alp.getNextLaunchParameter("a")); - Assert.assertEquals("3", alp.getNextLaunchParameter("a")); - Assert.assertEquals("3", alp.getNextLaunchParameter("a")); - - Assert.assertEquals("", alp.getNextLaunchParameter("b")); - Assert.assertEquals("5", alp.getNextLaunchParameter("b")); - Assert.assertEquals("", alp.getNextLaunchParameter("b")); - Assert.assertEquals("", alp.getNextLaunchParameter("b")); - - alp = new AgentLaunchParameter("|:"); - Assert.assertEquals("", alp.getNextLaunchParameter("b")); - Assert.assertEquals("", alp.getNextLaunchParameter("a")); - - alp = new AgentLaunchParameter("HBASE_MASTER:a,b:DO_NOT_REGISTER:"); - Assert.assertEquals("a,b", alp.getNextLaunchParameter("HBASE_MASTER")); - Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER")); - Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER")); - - alp = new AgentLaunchParameter("HBASE_MASTER:a,b:DO_NOT_REGISTER::c:::"); - Assert.assertEquals("a,b", alp.getNextLaunchParameter("HBASE_MASTER")); - Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER")); - Assert.assertEquals("", alp.getNextLaunchParameter("HBASE_MASTER")); - Assert.assertEquals("c", alp.getNextLaunchParameter("HBASE_MASTER")); - Assert.assertEquals("c", alp.getNextLaunchParameter("HBASE_MASTER")); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java deleted file mode 100644 index 5e1dc7fd60a..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.providers.agent.application.metadata.Metainfo; -import org.apache.slider.tools.TestUtility; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TestAgentUtils { - protected static final Logger log = - LoggerFactory.getLogger(TestAgentUtils.class); - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - private static final String metainfo_str = "\n" - + " 2.0\n" - + " \n" - + " MYTESTAPPLICATION\n" - + " \n" - + " My Test Application\n" - + " \n" - + " 1.0\n" - + " YARN-APP\n" - + " \n" - + " \n" - + " REST\n" - + " MASTER\n" - + " \n" - + " \n" - + " PYTHON\n" - + " 600\n" - + " \n" - + " \n" - + " \n" - + " \n" - + ""; - - @Test - public void testGetApplicationMetainfo() throws Exception { - String zipFileName = TestUtility.createAppPackage( - folder, - "testpkg", - "test.zip", - "target/test-classes/org/apache/slider/common/tools/test"); - Configuration configuration = new Configuration(); - FileSystem fs = FileSystem.getLocal(configuration); - log.info("fs working dir is {}", fs.getWorkingDirectory().toString()); - SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); - - // Without accompany metainfo file, read metainfo from the zip file - Metainfo metainfo = AgentUtils.getApplicationMetainfo( - sliderFileSystem, zipFileName, false); - Assert.assertNotNull(metainfo.getApplication()); - Assert.assertEquals("STORM", metainfo.getApplication().getName()); - - // With accompany metainfo file, read metainfo from the accompany file - String acompanyFileName = zipFileName + ".metainfo.xml"; - File f = new File(acompanyFileName); - try (BufferedWriter writer = new BufferedWriter(new FileWriter(f))) { - writer.write(metainfo_str); - } - metainfo = AgentUtils.getApplicationMetainfo( - sliderFileSystem, zipFileName, false); - Assert.assertNotNull(metainfo.getApplication()); - Assert.assertEquals("MYTESTAPPLICATION", metainfo.getApplication().getName()); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java deleted file mode 100644 index dedf4f67396..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent; - -import com.google.common.io.Files; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.slider.common.params.ActionCreateArgs; -import org.apache.slider.common.params.AddonArgsDelegate; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.core.conf.ConfTree; -import org.apache.slider.core.conf.ConfTreeOperations; -import org.apache.slider.core.exceptions.BadConfigException; -import org.apache.slider.core.persist.AppDefinitionPersister; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.List; - -/** - * - */ -public class TestAppDefinitionPersister { - protected static final Logger log = - LoggerFactory.getLogger(TestAppDefinitionPersister.class); - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - /** - * @BeforeClass public static void initialize() { BasicConfigurator.resetConfiguration(); - * BasicConfigurator.configure(); }* - */ - - - @Test - public void testAppDefinitionPersister() throws Exception { - Configuration configuration = new Configuration(); - FileSystem fs = FileSystem.getLocal(configuration); - log.info("fs working dir is {}", fs.getWorkingDirectory().toString()); - SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); - - AppDefinitionPersister adp = new AppDefinitionPersister(sliderFileSystem); - String clustername = "c1"; - ActionCreateArgs buildInfo = new ActionCreateArgs(); - buildInfo.appMetaInfo = null; - buildInfo.appDef = null; - buildInfo.addonDelegate = new AddonArgsDelegate(); - - // nothing to do - adp.processSuppliedDefinitions(clustername, buildInfo, null); - adp.persistPackages(); - List appDefinitions = adp.getAppDefinitions(); - Assert.assertTrue(appDefinitions.size() == 0); - - ConfTree ct = new ConfTree(); - ConfTreeOperations appConf = new ConfTreeOperations(ct); - final File tempDir = Files.createTempDir(); - final File metainfo = new File(tempDir, "metainfo.json"); - - // unreadable metainfo - buildInfo.appMetaInfo = metainfo; - - try { - adp.processSuppliedDefinitions(clustername, buildInfo, appConf); - } catch (BadConfigException bce) { - log.info(bce.getMessage()); - Assert.assertTrue(bce.getMessage().contains( - "Path specified with " - + "--metainfo either cannot be read or is not a file")); - } - - try (PrintWriter writer = new PrintWriter(metainfo.getAbsolutePath(), "UTF-8")) { - writer.println("{"); - writer.println("}"); - } - buildInfo.appDef = metainfo; - - try { - adp.processSuppliedDefinitions(clustername, buildInfo, appConf); - } catch (BadConfigException bce) { - log.info(bce.getMessage()); - Assert.assertTrue(bce.getMessage().contains( - "Both --metainfo and --appdef cannot be specified")); - } - - // both --metainfojson and --appdef cannot be specified - buildInfo.appMetaInfo = null; - buildInfo.appMetaInfoJson = "{}"; - try { - adp.processSuppliedDefinitions(clustername, buildInfo, appConf); - } catch (BadConfigException bce) { - log.info(bce.getMessage()); - Assert.assertTrue(bce.getMessage().contains( - "Both --metainfojson and --appdef cannot be specified")); - } - - buildInfo.appDef = null; - - buildInfo.appMetaInfoJson = ""; - try { - adp.processSuppliedDefinitions(clustername, buildInfo, appConf); - } catch (BadConfigException bce) { - log.info(bce.getMessage()); - Assert.assertTrue(bce.getMessage().contains( - "Empty string specified with --metainfojson")); - } - buildInfo.appMetaInfo = metainfo; - - // both --metainfo and --metainfojson cannot be specified - buildInfo.appMetaInfoJson = "{}"; - try { - adp.processSuppliedDefinitions(clustername, buildInfo, appConf); - } catch (BadConfigException bce) { - log.info(bce.getMessage()); - Assert.assertTrue(bce.getMessage().contains( - "Both --metainfo and --metainfojson cannot be specified")); - } - buildInfo.appMetaInfoJson = null; - - appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath()); - - try { - adp.processSuppliedDefinitions(clustername, buildInfo, appConf); - } catch (BadConfigException bce) { - log.info(bce.getMessage()); - Assert.assertTrue(bce.getMessage().contains( - "application.def cannot " - + "not be set if --metainfo is specified in the cmd line")); - } - - appConf.getGlobalOptions().remove(AgentKeys.APP_DEF); - - adp.processSuppliedDefinitions(clustername, buildInfo, appConf); - appDefinitions = adp.getAppDefinitions(); - Assert.assertTrue(appDefinitions.size() == 1); - Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip")); - log.info(appDefinitions.get(0).toString()); - Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith("default")); - Assert.assertTrue(appDefinitions.get(0).targetFolderInFs.toString().contains("cluster/c1/appdef")); - Assert.assertEquals("appPkg.zip", appDefinitions.get(0).pkgName); - - buildInfo.appDef = tempDir; - buildInfo.appMetaInfo = null; - - appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath()); - - try { - adp.processSuppliedDefinitions(clustername, buildInfo, appConf); - } catch (BadConfigException bce) { - log.info(bce.getMessage()); - Assert.assertTrue(bce.getMessage().contains("application.def must not be set if --appdef is provided")); - } - - adp.getAppDefinitions().clear(); - appConf.getGlobalOptions().remove(AgentKeys.APP_DEF); - adp.processSuppliedDefinitions(clustername, buildInfo, appConf); - appDefinitions = adp.getAppDefinitions(); - Assert.assertTrue(appDefinitions.size() == 1); - Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip")); - log.info(appDefinitions.get(0).toString()); - Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith(tempDir.toString())); - Assert.assertTrue(appDefinitions.get(0).targetFolderInFs.toString().contains("cluster/c1/appdef")); - Assert.assertEquals("appPkg.zip", appDefinitions.get(0).pkgName); - - adp.getAppDefinitions().clear(); - buildInfo.appDef = null; - buildInfo.appMetaInfo = null; - appConf.getGlobalOptions().remove(AgentKeys.APP_DEF); - - ArrayList list = new ArrayList() {{ - add("addon1"); - add(""); - add("addon2"); - add(metainfo.getAbsolutePath()); - }}; - - buildInfo.addonDelegate.addonTuples = list; - try { - adp.processSuppliedDefinitions(clustername, buildInfo, appConf); - } catch (BadConfigException bce) { - log.info(bce.getMessage()); - Assert.assertTrue(bce.getMessage().contains("addon package can only be specified if main app package is specified")); - } - - buildInfo.appMetaInfo = metainfo; - - try { - adp.processSuppliedDefinitions(clustername, buildInfo, appConf); - } catch (BadConfigException bce) { - log.info(bce.getMessage()); - Assert.assertTrue(bce.getMessage().contains("Invalid path for addon package addon1")); - } - - appConf.getGlobalOptions().remove(AgentKeys.APP_DEF); - - list = new ArrayList() {{ - add("addon1"); - add(tempDir.getAbsolutePath()); - add("addon2"); - add(metainfo.getAbsolutePath()); - }}; - - buildInfo.addonDelegate.addonTuples = list; - adp.getAppDefinitions().clear(); - - adp.processSuppliedDefinitions(clustername, buildInfo, appConf); - appDefinitions = adp.getAppDefinitions(); - - Assert.assertTrue(appDefinitions.size() == 3); - Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip")); - Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon1").contains( - "addons/addon1/addon_addon1.zip")); - Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon2").contains( - "addons/addon2/addon_addon2.zip")); - log.info(appConf.getGlobalOptions().get("application.addons")); - Assert.assertTrue(appConf.getGlobalOptions().get("application.addons").contains( - "application.addon.addon2,application.addon.addon1") - || appConf.getGlobalOptions().get("application.addons").contains( - "application.addon.addon1,application.addon.addon2")); - int seen = 0; - for (AppDefinitionPersister.AppDefinition adp_ad : appDefinitions) { - if (adp_ad.pkgName.equals("appPkg.zip")) { - log.info(adp_ad.toString()); - Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith("default")); - Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("cluster/c1/appdef")); - seen++; - } - if (adp_ad.pkgName.equals("addon_addon1.zip")) { - log.info(adp_ad.toString()); - Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith(tempDir.toString())); - Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("addons/addon1")); - seen++; - } - if (adp_ad.pkgName.equals("addon_addon2.zip")) { - log.info(adp_ad.toString()); - Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith("metainfo.json")); - Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("addons/addon2")); - seen++; - } - } - Assert.assertEquals(3, seen); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java deleted file mode 100644 index 7b38ee3906e..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.providers.agent; - -import org.junit.Assert; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class TestComponentTagProvider { - protected static final Logger log = - LoggerFactory.getLogger(TestComponentTagProvider.class); - - @Test - public void testTagProvider() throws Exception { - ComponentTagProvider ctp = new ComponentTagProvider(); - Assert.assertEquals("", ctp.getTag(null, null)); - Assert.assertEquals("", ctp.getTag(null, "cid")); - Assert.assertEquals("", ctp.getTag("comp1", null)); - - Assert.assertEquals("1", ctp.getTag("comp1", "cid1")); - Assert.assertEquals("2", ctp.getTag("comp1", "cid2")); - Assert.assertEquals("3", ctp.getTag("comp1", "cid3")); - ctp.releaseTag("comp1", "cid2"); - Assert.assertEquals("2", ctp.getTag("comp1", "cid22")); - - ctp.releaseTag("comp1", "cid4"); - ctp.recordAssignedTag("comp1", "cid5", "5"); - Assert.assertEquals("4", ctp.getTag("comp1", "cid4")); - Assert.assertEquals("4", ctp.getTag("comp1", "cid4")); - Assert.assertEquals("6", ctp.getTag("comp1", "cid6")); - - ctp.recordAssignedTag("comp1", "cid55", "5"); - Assert.assertEquals("5", ctp.getTag("comp1", "cid55")); - - ctp.recordAssignedTag("comp2", "cidb3", "3"); - Assert.assertEquals("1", ctp.getTag("comp2", "cidb1")); - Assert.assertEquals("2", ctp.getTag("comp2", "cidb2")); - Assert.assertEquals("4", ctp.getTag("comp2", "cidb4")); - - ctp.recordAssignedTag("comp2", "cidb5", "six"); - ctp.recordAssignedTag("comp2", "cidb5", "-55"); - ctp.recordAssignedTag("comp2", "cidb5", "tags"); - ctp.recordAssignedTag("comp2", "cidb5", null); - ctp.recordAssignedTag("comp2", "cidb5", ""); - ctp.recordAssignedTag("comp2", "cidb5", "5"); - Assert.assertEquals("6", ctp.getTag("comp2", "cidb6")); - - ctp.recordAssignedTag("comp2", null, "5"); - ctp.recordAssignedTag(null, null, "5"); - ctp.releaseTag("comp1", null); - ctp.releaseTag(null, "cid4"); - ctp.releaseTag(null, null); - } - - @Test - public void testTagProviderWithThread() throws Exception { - ComponentTagProvider ctp = new ComponentTagProvider(); - Thread thread = new Thread(new Taggged(ctp)); - Thread thread2 = new Thread(new Taggged(ctp)); - Thread thread3 = new Thread(new Taggged(ctp)); - thread.start(); - thread2.start(); - thread3.start(); - ctp.getTag("comp1", "cid50"); - thread.join(); - thread2.join(); - thread3.join(); - Assert.assertEquals("101", ctp.getTag("comp1", "cid101")); - } - - public class Taggged implements Runnable { - private final ComponentTagProvider ctp; - - public Taggged(ComponentTagProvider ctp) { - this.ctp = ctp; - } - - public void run() { - for (int i = 0; i < 100; i++) { - String containerId = "cid" + (i + 1); - this.ctp.getTag("comp1", containerId); - } - for (int i = 0; i < 100; i++) { - String containerId = "cid" + (i + 1); - this.ctp.getTag("comp1", containerId); - } - for (int i = 0; i < 100; i += 2) { - String containerId = "cid" + (i + 1); - this.ctp.releaseTag("comp1", containerId); - } - for (int i = 0; i < 100; i += 2) { - String containerId = "cid" + (i + 1); - this.ctp.getTag("comp1", containerId); - } - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java deleted file mode 100644 index 6a2e5ab59a8..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent; - -import org.junit.Assert; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TestState { - protected static final Logger log = LoggerFactory.getLogger(TestState.class); - - @Test - public void testState() throws Exception { - State state = State.STARTED; - Assert.assertEquals(Command.STOP, state.getSupportedCommand(false, true)); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java deleted file mode 100644 index 3aa44a1cf40..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.junit.Assert; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; - -/** - * - */ -public class TestConfigParser { - protected static final Logger log = - LoggerFactory.getLogger(TestConfigParser.class); - private static final String config_1_str = "\n" - + " \n" - + " security.client.protocol.acl\n" - + " *\n" - + " ACL for HRegionInterface protocol implementations (ie. \n" - + " clients talking to HRegionServers)\n" - + " The ACL is a comma-separated list of user and group names. The user and \n" - + " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n" - + " A special value of \"*\" means all users are allowed.\n" - + " \n" - + "\n" - + " \n" - + " security.admin.protocol.acl\n" - + " *\n" - + " ACL for HMasterInterface protocol implementation (ie. \n" - + " clients talking to HMaster for admin operations).\n" - + " The ACL is a comma-separated list of user and group names. The user and \n" - + " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n" - + " A special value of \"*\" means all users are allowed.\n" - + " \n" - + "\n" - + " \n" - + " security.masterregion.protocol.acl\n" - + " *\n" - + " ACL for HMasterRegionInterface protocol implementations\n" - + " (for HRegionServers communicating with HMaster)\n" - + " The ACL is a comma-separated list of user and group names. The user and \n" - + " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n" - + " A special value of \"*\" means all users are allowed.\n" - + " \n" - + " \n" - + " emptyVal\n" - + " \n" - + " non-empty-desc\n" - + " \n" - + " \n" - + " emptyDesc\n" - + " \n" - + " \n" - + " \n" - + " \n" - + " noDesc\n" - + " \n" - + " \n" - + ""; - - @Test - public void testParse() throws IOException { - - InputStream config_1 = new ByteArrayInputStream(config_1_str.getBytes()); - DefaultConfig config = new DefaultConfigParser().parse(config_1); - Assert.assertNotNull(config); - Assert.assertNotNull(config.getPropertyInfos()); - Assert.assertEquals(6, config.getPropertyInfos().size()); - for (PropertyInfo pInfo : config.getPropertyInfos()) { - if (pInfo.getName().equals("security.client.protocol.acl")) { - Assert.assertEquals("*", pInfo.getValue()); - Assert.assertTrue(pInfo.getDescription().startsWith("ACL for HRegionInterface ")); - } - if (pInfo.getName().equals("emptyVal")) { - Assert.assertEquals("", pInfo.getValue()); - Assert.assertEquals("non-empty-desc", pInfo.getDescription()); - } - if (pInfo.getName().equals("emptyDesc")) { - Assert.assertEquals("", pInfo.getValue()); - Assert.assertEquals("", pInfo.getDescription()); - } - if (pInfo.getName().equals("noDesc")) { - Assert.assertEquals("", pInfo.getValue()); - Assert.assertNull(pInfo.getDescription()); - } - } - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java deleted file mode 100644 index ba1912a22f0..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.providers.agent.application.metadata; - -import org.apache.slider.providers.agent.AgentProviderService; -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Mockito; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; - -import static org.mockito.Mockito.doReturn; - -/** - * - */ -public class TestMetainfoParser { - protected static final Logger log = - LoggerFactory.getLogger(TestMetainfoParser.class); - public static final String METAINFO_XML = - "/org/apache/slider/providers/agent/application/metadata/metainfo.xml"; - - @Test - public void testParse() throws IOException { - - InputStream resStream = this.getClass().getResourceAsStream( - METAINFO_XML); - MetainfoParser parser = new MetainfoParser(); - Metainfo metainfo = parser.fromXmlStream(resStream); - Assert.assertNotNull(metainfo); - Assert.assertNotNull(metainfo.getApplication()); - Application application = metainfo.getApplication(); - assert "STORM".equals(application.getName()); - assert 6 == application.getComponents().size(); - OSPackage pkg = application.getOSSpecifics().get(0).getPackages().get(0); - assert "tarball".equals(pkg.getType()); - assert "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz".equals(pkg.getName()); - boolean found = false; - for (Component comp : application.getComponents()) { - if (comp != null && comp.getName().equals("NIMBUS")) { - found = true; - Assert.assertEquals(0, comp.getComponentExports().size()); - } - if (comp != null && comp.getName().equals("SUPERVISOR")) { - Assert.assertEquals(1, comp.getComponentExports().size()); - } - if (comp != null && comp.getName().equals("ANOTHER_COMPONENT")) { - assert 2 == comp.getCommands().size(); - assert "start command".equals(comp.getCommands().get(0).getExec()); - assert "START".equals(comp.getCommands().get(0).getName()); - assert "stop command".equals(comp.getCommands().get(1).getExec()); - assert "STOP".equals(comp.getCommands().get(1).getName()); - } - } - assert found; - Assert.assertEquals(0, application.getConfigFiles().size()); - assert 1 == application.getPackages().size(); - Package p = application.getPackages().get(0); - assert "tarball".equals(p.getType()); - assert "test-tarball-name.tgz".equals(p.getName()); - } - - @Test - public void testJsonParse() throws IOException { - String metaInfo1_json = "{\n" - + "\"schemaVersion\":\"2.2\",\n" - + "\"application\":{\n" - + "\"name\": \"MEMCACHED\"," - + "\"exportGroups\": [" - + "{" - + "\"name\": \"Servers\"," - + "\"exports\": [" - + "{" - + "\"name\": \"host_port\"," - + "\"value\": \"${MEMCACHED_HOST}:${site.global.port}\"" - + "}" - + "]" - + "}" - + "]," - + "\"components\": [" - + "{" - + "\"name\": \"MEMCACHED\"," - + "\"compExports\": \"Servers-host_port\"," - + "\"commands\": [" - + "{" - + "\"exec\": \"java -classpath /usr/myapps/memcached/*:/usr/lib/hadoop/lib/* com.thimbleware.jmemcached.Main\"" - + "}" - + "]" - + "}," - + "{" - + "\"name\": \"MEMCACHED2\"," - + "\"commands\": [" - + "{" - + "\"exec\": \"scripts/config.py\"," - + "\"type\": \"PYTHON\"," - + "\"name\": \"CONFIGURE\"" - + "}" - + "]," - + "\"dockerContainers\": [" - + "{" - + "\"name\": \"redis\"," - + "\"image\": \"dockerhub/redis\"," - + "\"options\": \"-net=bridge\"," - + "\"mounts\": [" - + "{" - + "\"containerMount\": \"/tmp/conf\"," - + "\"hostMount\": \"{$conf:@//site/global/app_root}/conf\"" - + "}" - + "]" - + "}" - + "]" - + "}" - + "]" - + "}" - + "}"; - - MetainfoParser parser = new MetainfoParser(); - Metainfo mInfo = parser.fromJsonString(metaInfo1_json); - Assert.assertEquals("2.2", mInfo.getSchemaVersion()); - - Application app = mInfo.getApplication(); - Assert.assertNotNull(app); - - Assert.assertEquals("MEMCACHED", app.getName()); - List egs = app.getExportGroups(); - Assert.assertEquals(1, egs.size()); - ExportGroup eg = egs.get(0); - Assert.assertEquals("Servers", eg.getName()); - List exports = eg.getExports(); - Assert.assertEquals(1, exports.size()); - Export export = exports.get(0); - Assert.assertEquals("host_port", export.getName()); - Assert.assertEquals("${MEMCACHED_HOST}:${site.global.port}", export.getValue()); - - List components = app.getComponents(); - Assert.assertEquals(2, components.size()); - - Component c1 = mInfo.getApplicationComponent("MEMCACHED"); - Assert.assertNotNull(c1); - Assert.assertEquals("MEMCACHED", c1.getName()); - Assert.assertEquals("Servers-host_port", c1.getCompExports()); - Assert.assertEquals(1, c1.getCommands().size()); - ComponentCommand cmd = c1.getCommands().get(0); - Assert.assertEquals("START", cmd.getName()); - Assert.assertEquals("SHELL", cmd.getType()); - Assert.assertEquals("java -classpath /usr/myapps/memcached/*:/usr/lib/hadoop/lib/* com.thimbleware.jmemcached.Main", - cmd.getExec()); - - Component c2 = mInfo.getApplicationComponent("MEMCACHED2"); - Assert.assertNotNull(c2); - Assert.assertEquals("MEMCACHED2", c2.getName()); - Assert.assertEquals(1, c2.getCommands().size()); - cmd = c2.getCommands().get(0); - Assert.assertEquals("CONFIGURE", cmd.getName()); - Assert.assertEquals("PYTHON", cmd.getType()); - Assert.assertEquals("scripts/config.py", cmd.getExec()); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java deleted file mode 100644 index 7fceac74f7c..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.publisher; - -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.slider.common.tools.SliderFileSystem; -import org.apache.slider.providers.agent.AgentProviderService; -import org.apache.slider.server.appmaster.actions.QueueAccess; -import org.apache.slider.server.appmaster.state.StateAccessForProviders; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * - */ -public class TestAgentProviderService extends AgentProviderService { - protected static final Logger log = - LoggerFactory.getLogger(TestAgentProviderService.class); - - public TestAgentProviderService() { - super(); - log.info("TestAgentProviderService created"); - } - - @Override - public void bind(StateAccessForProviders stateAccessor, - QueueAccess queueAccess, - List liveContainers) { - super.bind(stateAccessor, queueAccess, liveContainers); - Map dummyProps = new HashMap(); - dummyProps.put("prop1", "val1"); - dummyProps.put("prop2", "val2"); - log.info("publishing dummy-site.xml with values {}", dummyProps); - publishApplicationInstanceData("dummy-site", "dummy configuration", - dummyProps.entrySet()); - // publishing global config for testing purposes - publishApplicationInstanceData("global", "global configuration", - stateAccessor.getAppConfSnapshot() - .getGlobalOptions().entrySet()); - } - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java deleted file mode 100644 index f49e15a5bf3..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.slider.server.appmaster.web.rest.publisher; - -import org.apache.slider.providers.ProviderService; -import org.apache.slider.providers.agent.AgentProviderFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * - */ -public class TestSliderProviderFactory extends AgentProviderFactory{ - protected static final Logger log = - LoggerFactory.getLogger(TestSliderProviderFactory.class); - - public TestSliderProviderFactory() { - log.info("Created TestSliderProviderFactory"); - } - - @Override - public ProviderService createServerProvider() { - log.info("Creating TestAgentProviderService"); - return new TestAgentProviderService(); - } -}