YARN-5690. Integrate native services modules into maven build. Contributed by Billie Rinaldi
This commit is contained in:
parent
ef5a3628c2
commit
09e4b9e897
|
@ -86,6 +86,32 @@
|
||||||
<include>*-sources.jar</include>
|
<include>*-sources.jar</include>
|
||||||
</includes>
|
</includes>
|
||||||
</fileSet>
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target</directory>
|
||||||
|
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
|
||||||
|
<includes>
|
||||||
|
<include>*-sources.jar</include>
|
||||||
|
</includes>
|
||||||
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf</directory>
|
||||||
|
<outputDirectory>etc/hadoop</outputDirectory>
|
||||||
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target/hadoop-yarn-slider-core-${project.version}</directory>
|
||||||
|
<outputDirectory>/share/hadoop/${hadoop.component}/lib/slider</outputDirectory>
|
||||||
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target</directory>
|
||||||
|
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
|
||||||
|
<includes>
|
||||||
|
<include>*-sources.jar</include>
|
||||||
|
</includes>
|
||||||
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target/hadoop-yarn-services-api-${project.version}</directory>
|
||||||
|
<outputDirectory>/share/hadoop/${hadoop.component}/lib/services-api</outputDirectory>
|
||||||
|
</fileSet>
|
||||||
<fileSet>
|
<fileSet>
|
||||||
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target</directory>
|
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target</directory>
|
||||||
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
|
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
|
||||||
|
|
|
@ -0,0 +1,36 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.01
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||||
|
<id>hadoop-yarn-services-api-dist</id>
|
||||||
|
<formats>
|
||||||
|
<format>dir</format>
|
||||||
|
</formats>
|
||||||
|
<includeBaseDirectory>false</includeBaseDirectory>
|
||||||
|
<dependencySets>
|
||||||
|
<dependencySet>
|
||||||
|
<useProjectArtifact>false</useProjectArtifact>
|
||||||
|
<includes>
|
||||||
|
<include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base</include>
|
||||||
|
<include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider</include>
|
||||||
|
<include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations</include>
|
||||||
|
<include>io.swagger:swagger-annotations</include>
|
||||||
|
</includes>
|
||||||
|
</dependencySet>
|
||||||
|
</dependencySets>
|
||||||
|
</assembly>
|
|
@ -0,0 +1,30 @@
|
||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
contributor license agreements. See the NOTICE file distributed with
|
||||||
|
this work for additional information regarding copyright ownership.
|
||||||
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
(the "License"); you may not use this file except in compliance with
|
||||||
|
the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.01
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
-->
|
||||||
|
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
|
||||||
|
<id>hadoop-yarn-slider-dist</id>
|
||||||
|
<formats>
|
||||||
|
<format>dir</format>
|
||||||
|
</formats>
|
||||||
|
<includeBaseDirectory>false</includeBaseDirectory>
|
||||||
|
<dependencySets>
|
||||||
|
<dependencySet>
|
||||||
|
<useProjectArtifact>false</useProjectArtifact>
|
||||||
|
</dependencySet>
|
||||||
|
</dependencySets>
|
||||||
|
</assembly>
|
|
@ -425,6 +425,12 @@
|
||||||
<version>${project.version}</version>
|
<version>${project.version}</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-yarn-slider-core</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.hadoop</groupId>
|
<groupId>org.apache.hadoop</groupId>
|
||||||
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
|
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
|
||||||
|
|
|
@ -49,7 +49,9 @@ function hadoop_usage
|
||||||
hadoop_add_subcommand "router" daemon "run the Router daemon"
|
hadoop_add_subcommand "router" daemon "run the Router daemon"
|
||||||
hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
|
hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration"
|
||||||
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
|
hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
|
||||||
|
hadoop_add_subcommand "services-api" "run slider services api"
|
||||||
hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
|
hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon"
|
||||||
|
hadoop_add_subcommand "slider" "run a slider app"
|
||||||
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
|
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
|
||||||
hadoop_add_subcommand "timelineserver" daemon "run the timeline server"
|
hadoop_add_subcommand "timelineserver" daemon "run the timeline server"
|
||||||
hadoop_add_subcommand "top" client "view cluster information"
|
hadoop_add_subcommand "top" client "view cluster information"
|
||||||
|
@ -149,10 +151,38 @@ function yarncmd_case
|
||||||
scmadmin)
|
scmadmin)
|
||||||
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
|
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
|
||||||
;;
|
;;
|
||||||
|
services-api)
|
||||||
|
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
|
||||||
|
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
|
||||||
|
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*'
|
||||||
|
HADOOP_CLASSNAME='org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp'
|
||||||
|
hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
|
||||||
|
HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \
|
||||||
|
-Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\
|
||||||
|
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
|
||||||
|
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
|
||||||
|
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
|
||||||
|
${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
|
||||||
|
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
|
||||||
|
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
|
||||||
|
;;
|
||||||
sharedcachemanager)
|
sharedcachemanager)
|
||||||
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
|
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
|
||||||
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
|
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
|
||||||
;;
|
;;
|
||||||
|
slider)
|
||||||
|
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
|
||||||
|
HADOOP_CLASSNAME='org.apache.slider.Slider'
|
||||||
|
hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
|
||||||
|
HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \
|
||||||
|
-Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\
|
||||||
|
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
|
||||||
|
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
|
||||||
|
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
|
||||||
|
${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
|
||||||
|
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
|
||||||
|
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
|
||||||
|
;;
|
||||||
timelinereader)
|
timelinereader)
|
||||||
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
|
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
|
||||||
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
|
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
|
||||||
|
|
|
@ -159,12 +159,6 @@
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.fasterxml.jackson.jaxrs</groupId>
|
<groupId>com.fasterxml.jackson.jaxrs</groupId>
|
||||||
<artifactId>jackson-jaxrs-json-provider</artifactId>
|
<artifactId>jackson-jaxrs-json-provider</artifactId>
|
||||||
<exclusions>
|
|
||||||
<exclusion>
|
|
||||||
<groupId>com.fasterxml.jackson.jaxrs</groupId>
|
|
||||||
<artifactId>jackson-jaxrs-base</artifactId>
|
|
||||||
</exclusion>
|
|
||||||
</exclusions>
|
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.mockito</groupId>
|
<groupId>org.mockito</groupId>
|
||||||
|
@ -192,6 +186,44 @@
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
<profiles>
|
<profiles>
|
||||||
|
<profile>
|
||||||
|
<id>dist</id>
|
||||||
|
<activation>
|
||||||
|
<activeByDefault>false</activeByDefault>
|
||||||
|
</activation>
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-assemblies</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>dist</id>
|
||||||
|
<phase>prepare-package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>single</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<appendAssemblyId>false</appendAssemblyId>
|
||||||
|
<attach>false</attach>
|
||||||
|
<finalName>${project.artifactId}-${project.version}</finalName>
|
||||||
|
<descriptorRefs>
|
||||||
|
<descriptorRef>hadoop-yarn-services-api</descriptorRef>
|
||||||
|
</descriptorRefs>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</profile>
|
||||||
|
|
||||||
<profile>
|
<profile>
|
||||||
<id>rat</id>
|
<id>rat</id>
|
||||||
|
|
|
@ -0,0 +1,68 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
# This is the log4j configuration for Slider Application Master
|
||||||
|
|
||||||
|
# Log rotation based on size (100KB) with a max of 10 backup files
|
||||||
|
log4j.rootLogger=INFO, amlog
|
||||||
|
log4j.threshhold=ALL
|
||||||
|
log4j.appender.amlog=org.apache.log4j.RollingFileAppender
|
||||||
|
log4j.appender.amlog.layout=org.apache.log4j.PatternLayout
|
||||||
|
log4j.appender.amlog.File=${LOG_DIR}/slider.log
|
||||||
|
log4j.appender.amlog.MaxFileSize=1MB
|
||||||
|
log4j.appender.amlog.MaxBackupIndex=10
|
||||||
|
|
||||||
|
# log layout skips stack-trace creation operations by avoiding line numbers and method
|
||||||
|
log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
|
||||||
|
|
||||||
|
# debug edition is much more expensive
|
||||||
|
#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
|
||||||
|
|
||||||
|
# configure stderr
|
||||||
|
# set the conversion pattern of stderr
|
||||||
|
# Print the date in ISO 8601 format
|
||||||
|
log4j.appender.stderr=org.apache.log4j.ConsoleAppender
|
||||||
|
log4j.appender.stderr.Target=System.err
|
||||||
|
log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
|
||||||
|
log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
|
||||||
|
|
||||||
|
log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
|
||||||
|
log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
|
||||||
|
log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
|
||||||
|
#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
|
||||||
|
|
||||||
|
# for debugging Slider
|
||||||
|
#log4j.logger.org.apache.slider=DEBUG
|
||||||
|
|
||||||
|
# uncomment to debug service lifecycle issues
|
||||||
|
#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
|
||||||
|
#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
|
||||||
|
|
||||||
|
# uncomment for YARN operations
|
||||||
|
#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
|
||||||
|
|
||||||
|
# uncomment this to debug security problems
|
||||||
|
#log4j.logger.org.apache.hadoop.security=DEBUG
|
||||||
|
|
||||||
|
#crank back on some noise
|
||||||
|
log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
|
||||||
|
log4j.logger.org.apache.hadoop.hdfs=WARN
|
||||||
|
log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR
|
||||||
|
|
||||||
|
log4j.logger.org.apache.zookeeper=WARN
|
||||||
|
log4j.logger.org.apache.curator.framework.state=ERROR
|
||||||
|
log4j.logger.org.apache.curator.framework.imps=WARN
|
|
@ -331,6 +331,44 @@
|
||||||
|
|
||||||
|
|
||||||
<profiles>
|
<profiles>
|
||||||
|
<profile>
|
||||||
|
<id>dist</id>
|
||||||
|
<activation>
|
||||||
|
<activeByDefault>false</activeByDefault>
|
||||||
|
</activation>
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.hadoop</groupId>
|
||||||
|
<artifactId>hadoop-assemblies</artifactId>
|
||||||
|
<version>${project.version}</version>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>dist</id>
|
||||||
|
<phase>prepare-package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>single</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<appendAssemblyId>false</appendAssemblyId>
|
||||||
|
<attach>false</attach>
|
||||||
|
<finalName>${project.artifactId}-${project.version}</finalName>
|
||||||
|
<descriptorRefs>
|
||||||
|
<descriptorRef>hadoop-yarn-slider-dist</descriptorRef>
|
||||||
|
</descriptorRefs>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</profile>
|
||||||
<profile>
|
<profile>
|
||||||
<id>compile-protobuf</id>
|
<id>compile-protobuf</id>
|
||||||
<build>
|
<build>
|
||||||
|
|
|
@ -2162,6 +2162,24 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
|
||||||
new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME);
|
new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME);
|
||||||
hasServerLog4jProperties = log4jserver.isFile();
|
hasServerLog4jProperties = log4jserver.isFile();
|
||||||
}
|
}
|
||||||
|
if (!hasServerLog4jProperties) {
|
||||||
|
// check for log4j properties in hadoop conf dir
|
||||||
|
String hadoopConfDir = System.getenv(ApplicationConstants.Environment
|
||||||
|
.HADOOP_CONF_DIR.name());
|
||||||
|
if (hadoopConfDir != null) {
|
||||||
|
File localFile = new File(hadoopConfDir, SliderKeys
|
||||||
|
.LOG4J_SERVER_PROP_FILENAME);
|
||||||
|
if (localFile.exists()) {
|
||||||
|
Path localFilePath = createLocalPath(localFile);
|
||||||
|
remoteConfPath = new Path(clusterDirectory,
|
||||||
|
SliderKeys.SUBMITTED_CONF_DIR);
|
||||||
|
Path remoteFilePath = new Path(remoteConfPath, SliderKeys
|
||||||
|
.LOG4J_SERVER_PROP_FILENAME);
|
||||||
|
copy(config, localFilePath, remoteFilePath);
|
||||||
|
hasServerLog4jProperties = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
// the assumption here is that minimr cluster => this is a test run
|
// the assumption here is that minimr cluster => this is a test run
|
||||||
// and the classpath can look after itself
|
// and the classpath can look after itself
|
||||||
|
|
||||||
|
@ -2300,7 +2318,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
|
||||||
// enable asserts
|
// enable asserts
|
||||||
commandLine.enableJavaAssertions();
|
commandLine.enableJavaAssertions();
|
||||||
|
|
||||||
// if the conf dir has a log4j-server.properties, switch to that
|
// if the conf dir has a slideram-log4j.properties, switch to that
|
||||||
if (hasServerLog4jProperties) {
|
if (hasServerLog4jProperties) {
|
||||||
commandLine.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME);
|
commandLine.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME);
|
||||||
commandLine.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
|
commandLine.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
|
||||||
|
@ -4471,14 +4489,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
|
||||||
return EXIT_SUCCESS;
|
return EXIT_SUCCESS;
|
||||||
}
|
}
|
||||||
|
|
||||||
String libDir = System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
|
String[] libDirs = SliderUtils.getLibDirs();
|
||||||
if (isSet(libDir)) {
|
if (libDirs.length > 0) {
|
||||||
File srcFolder = new File(libDir);
|
|
||||||
File tempLibTarGzipFile = File.createTempFile(
|
File tempLibTarGzipFile = File.createTempFile(
|
||||||
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME + "_",
|
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME + "_",
|
||||||
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT);
|
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT);
|
||||||
// copy all jars except slider-core-<version>.jar
|
// copy all jars
|
||||||
tarGzipFolder(srcFolder, tempLibTarGzipFile, createJarFilter());
|
tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter());
|
||||||
|
|
||||||
log.info("Uploading dependency for AM (version {}) from {} to {}",
|
log.info("Uploading dependency for AM (version {}) from {} to {}",
|
||||||
version, tempLibTarGzipFile.toURI(), dependencyLibTarGzip.toUri());
|
version, tempLibTarGzipFile.toURI(), dependencyLibTarGzip.toUri());
|
||||||
|
|
|
@ -182,7 +182,7 @@ public interface SliderKeys extends SliderXmlConfKeys {
|
||||||
/**
|
/**
|
||||||
* Slider AM log4j file name : {@value}
|
* Slider AM log4j file name : {@value}
|
||||||
*/
|
*/
|
||||||
String LOG4J_SERVER_PROP_FILENAME = "log4j-server.properties";
|
String LOG4J_SERVER_PROP_FILENAME = "slideram-log4j.properties";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Standard log4j file name : {@value}
|
* Standard log4j file name : {@value}
|
||||||
|
|
|
@ -25,6 +25,7 @@ import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
|
||||||
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
|
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
|
||||||
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
|
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
|
||||||
import org.apache.commons.io.output.ByteArrayOutputStream;
|
import org.apache.commons.io.output.ByteArrayOutputStream;
|
||||||
|
import org.apache.commons.lang.ArrayUtils;
|
||||||
import org.apache.commons.lang.StringUtils;
|
import org.apache.commons.lang.StringUtils;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
|
||||||
|
@ -1399,6 +1400,22 @@ public final class SliderUtils {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static String getLibDir() {
|
||||||
|
String[] libDirs = getLibDirs();
|
||||||
|
if (libDirs == null || libDirs.length == 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return libDirs[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String[] getLibDirs() {
|
||||||
|
String libDirStr = System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
|
||||||
|
if (isUnset(libDirStr)) {
|
||||||
|
return ArrayUtils.EMPTY_STRING_ARRAY;
|
||||||
|
}
|
||||||
|
return StringUtils.split(libDirStr, ',');
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Submit a JAR containing a specific class and map it
|
* Submit a JAR containing a specific class and map it
|
||||||
* @param providerResources provider map to build up
|
* @param providerResources provider map to build up
|
||||||
|
@ -1962,31 +1979,34 @@ public final class SliderUtils {
|
||||||
/**
|
/**
|
||||||
* Given a source folder create a tar.gz file
|
* Given a source folder create a tar.gz file
|
||||||
*
|
*
|
||||||
* @param srcFolder
|
* @param libDirs
|
||||||
* @param tarGzipFile
|
* @param tarGzipFile
|
||||||
*
|
*
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public static void tarGzipFolder(File srcFolder, File tarGzipFile,
|
public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
|
||||||
FilenameFilter filter) throws IOException {
|
FilenameFilter filter) throws IOException {
|
||||||
log.info("Tar-gzipping folder {} to {}", srcFolder.getAbsolutePath(),
|
log.info("Tar-gzipping folders {} to {}", libDirs,
|
||||||
tarGzipFile.getAbsolutePath());
|
tarGzipFile.getAbsolutePath());
|
||||||
List<String> files = new ArrayList<>();
|
|
||||||
generateFileList(files, srcFolder, srcFolder, true, filter);
|
|
||||||
|
|
||||||
try(TarArchiveOutputStream taos =
|
try(TarArchiveOutputStream taos =
|
||||||
new TarArchiveOutputStream(new GZIPOutputStream(
|
new TarArchiveOutputStream(new GZIPOutputStream(
|
||||||
new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
|
new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
|
||||||
for (String file : files) {
|
for (String libDir : libDirs) {
|
||||||
File srcFile = new File(srcFolder, file);
|
File srcFolder = new File(libDir);
|
||||||
TarArchiveEntry tarEntry = new TarArchiveEntry(
|
List<String> files = new ArrayList<>();
|
||||||
srcFile, file);
|
generateFileList(files, srcFolder, srcFolder, true, filter);
|
||||||
taos.putArchiveEntry(tarEntry);
|
for (String file : files) {
|
||||||
try(FileInputStream in = new FileInputStream(srcFile)) {
|
File srcFile = new File(srcFolder, file);
|
||||||
org.apache.commons.io.IOUtils.copy(in, taos);
|
TarArchiveEntry tarEntry = new TarArchiveEntry(
|
||||||
|
srcFile, file);
|
||||||
|
taos.putArchiveEntry(tarEntry);
|
||||||
|
try(FileInputStream in = new FileInputStream(srcFile)) {
|
||||||
|
org.apache.commons.io.IOUtils.copy(in, taos);
|
||||||
|
}
|
||||||
|
taos.flush();
|
||||||
|
taos.closeArchiveEntry();
|
||||||
}
|
}
|
||||||
taos.flush();
|
|
||||||
taos.closeArchiveEntry();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -562,8 +562,7 @@ public class AgentClientProvider extends AbstractClientProvider
|
||||||
}
|
}
|
||||||
|
|
||||||
private void expandAgentTar(File agentPkgDir) throws IOException {
|
private void expandAgentTar(File agentPkgDir) throws IOException {
|
||||||
String libDirProp =
|
String libDirProp = SliderUtils.getLibDir();
|
||||||
System.getProperty(PROPERTY_LIB_DIR);
|
|
||||||
File tarFile = new File(libDirProp, AGENT_TAR);
|
File tarFile = new File(libDirProp, AGENT_TAR);
|
||||||
expandTar(tarFile, agentPkgDir);
|
expandTar(tarFile, agentPkgDir);
|
||||||
}
|
}
|
||||||
|
|
|
@ -184,19 +184,20 @@ public class SliderAMClientProvider extends AbstractClientProvider
|
||||||
libdir,
|
libdir,
|
||||||
miniClusterTestRun);
|
miniClusterTestRun);
|
||||||
|
|
||||||
String libDirProp =
|
|
||||||
System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
|
|
||||||
log.info("Loading all dependencies for AM.");
|
log.info("Loading all dependencies for AM.");
|
||||||
// If slider.tar.gz is available in hdfs use it, else upload all jars
|
// If slider.tar.gz is available in hdfs use it, else upload all jars
|
||||||
Path dependencyLibTarGzip = fileSystem.getDependencyTarGzip();
|
Path dependencyLibTarGzip = fileSystem.getDependencyTarGzip();
|
||||||
if (fileSystem.isFile(dependencyLibTarGzip)) {
|
if (fileSystem.isFile(dependencyLibTarGzip)) {
|
||||||
SliderUtils.putAmTarGzipAndUpdate(providerResources, fileSystem);
|
SliderUtils.putAmTarGzipAndUpdate(providerResources, fileSystem);
|
||||||
} else {
|
} else {
|
||||||
ProviderUtils.addAllDependencyJars(providerResources,
|
for (String libDirProp : SliderUtils.getLibDirs()) {
|
||||||
fileSystem,
|
ProviderUtils.addAllDependencyJars(providerResources,
|
||||||
tempPath,
|
fileSystem,
|
||||||
libdir,
|
tempPath,
|
||||||
libDirProp);
|
libdir,
|
||||||
|
libDirProp);
|
||||||
|
|
||||||
|
}
|
||||||
}
|
}
|
||||||
addKeytabResourceIfNecessary(fileSystem,
|
addKeytabResourceIfNecessary(fileSystem,
|
||||||
instanceDescription,
|
instanceDescription,
|
||||||
|
|
|
@ -82,16 +82,14 @@ public class TestPublishedConfigurationOutputter {
|
||||||
|
|
||||||
String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
|
String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
|
||||||
"");
|
"");
|
||||||
assert output.contains(
|
assert output.contains("<name>key1</name><value>val1</value>");
|
||||||
"<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
|
|
||||||
|
|
||||||
File file = tmpDir.newFile();
|
File file = tmpDir.newFile();
|
||||||
configurationOutputter.save(file);
|
configurationOutputter.save(file);
|
||||||
|
|
||||||
assert FileUtils.readFileToString(file, Charsets.UTF_8)
|
assert FileUtils.readFileToString(file, Charsets.UTF_8)
|
||||||
.replaceAll("( |\\r|\\n)", "")
|
.replaceAll("( |\\r|\\n)", "")
|
||||||
.contains(
|
.contains("<name>key1</name><value>val1</value>");
|
||||||
"<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -103,14 +101,14 @@ public class TestPublishedConfigurationOutputter {
|
||||||
|
|
||||||
String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
|
String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
|
||||||
"");
|
"");
|
||||||
assert output.contains("<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
|
assert output.contains("<name>key1</name><value>val1</value>");
|
||||||
|
|
||||||
File file = tmpDir.newFile();
|
File file = tmpDir.newFile();
|
||||||
configurationOutputter.save(file);
|
configurationOutputter.save(file);
|
||||||
|
|
||||||
assert FileUtils.readFileToString(file, Charsets.UTF_8)
|
assert FileUtils.readFileToString(file, Charsets.UTF_8)
|
||||||
.replaceAll("( |\\r|\\n)", "")
|
.replaceAll("( |\\r|\\n)", "")
|
||||||
.contains( "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
|
.contains("<name>key1</name><value>val1</value>");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
Loading…
Reference in New Issue