From 09e4b9e897757a678e68f31606a69f62c54d2ca3 Mon Sep 17 00:00:00 2001 From: Gour Saha Date: Thu, 27 Oct 2016 08:50:36 -0700 Subject: [PATCH] YARN-5690. Integrate native services modules into maven build. Contributed by Billie Rinaldi --- .../resources/assemblies/hadoop-yarn-dist.xml | 26 +++++++ .../assemblies/hadoop-yarn-services-api.xml | 36 ++++++++++ .../assemblies/hadoop-yarn-slider-dist.xml | 30 ++++++++ hadoop-project/pom.xml | 6 ++ hadoop-yarn-project/hadoop-yarn/bin/yarn | 30 ++++++++ .../hadoop-yarn-services-api/pom.xml | 44 ++++++++++-- .../conf/slideram-log4j.properties | 68 +++++++++++++++++++ .../hadoop-yarn-slider-core/pom.xml | 38 +++++++++++ .../apache/slider/client/SliderClient.java | 29 ++++++-- .../org/apache/slider/common/SliderKeys.java | 2 +- .../slider/common/tools/SliderUtils.java | 48 +++++++++---- .../providers/agent/AgentClientProvider.java | 3 +- .../slideram/SliderAMClientProvider.java | 15 ++-- .../TestPublishedConfigurationOutputter.java | 10 ++- 14 files changed, 343 insertions(+), 42 deletions(-) create mode 100644 hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml create mode 100644 hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 289061f8add..4483fa87bb5 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -86,6 +86,32 @@ *-sources.jar + + hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target + /share/hadoop/${hadoop.component}/sources + + *-sources.jar + + + + hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf + etc/hadoop + + + hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target/hadoop-yarn-slider-core-${project.version} + /share/hadoop/${hadoop.component}/lib/slider + + + hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target + /share/hadoop/${hadoop.component}/sources + + *-sources.jar + + + + hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target/hadoop-yarn-services-api-${project.version} + /share/hadoop/${hadoop.component}/lib/services-api + hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target /share/hadoop/${hadoop.component}/sources diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml new file mode 100644 index 00000000000..589f7248485 --- /dev/null +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml @@ -0,0 +1,36 @@ + + + hadoop-yarn-services-api-dist + + dir + + false + + + false + + com.fasterxml.jackson.jaxrs:jackson-jaxrs-base + com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider + com.fasterxml.jackson.module:jackson-module-jaxb-annotations + io.swagger:swagger-annotations + + + + diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml new file mode 100644 index 00000000000..5de45a9c284 --- /dev/null +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml @@ -0,0 +1,30 @@ + + + hadoop-yarn-slider-dist + + dir + + false + + + false + + + diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 423bbc292c9..d803d13c937 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -425,6 +425,12 @@ ${project.version} + + org.apache.hadoop + hadoop-yarn-slider-core + ${project.version} + + org.apache.hadoop hadoop-mapreduce-client-jobclient diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 0f19989efc7..556843a7896 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -49,7 +49,9 @@ function hadoop_usage hadoop_add_subcommand "router" daemon "run the Router daemon" hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration" hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools" + hadoop_add_subcommand "services-api" "run slider services api" hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon" + hadoop_add_subcommand "slider" "run a slider app" hadoop_add_subcommand "timelinereader" client "run the timeline reader server" hadoop_add_subcommand "timelineserver" daemon "run the timeline server" hadoop_add_subcommand "top" client "view cluster information" @@ -149,10 +151,38 @@ function yarncmd_case scmadmin) HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin' ;; + services-api) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*' + hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*' + HADOOP_CLASSNAME='org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp' + hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS" + HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \ +-Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\ +${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\ +${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\ +${HADOOP_HDFS_HOME}/${HDFS_DIR},\ +${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\ +${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\ +${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" + ;; sharedcachemanager) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager' ;; + slider) + hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*' + HADOOP_CLASSNAME='org.apache.slider.Slider' + hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS" + HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \ +-Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\ +${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\ +${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\ +${HADOOP_HDFS_HOME}/${HDFS_DIR},\ +${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\ +${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\ +${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" + ;; timelinereader) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer' diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml index 78b7855ed11..c198f2839d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml @@ -159,12 +159,6 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-base - - org.mockito @@ -192,6 +186,44 @@ + + dist + + false + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.hadoop + hadoop-assemblies + ${project.version} + + + + + dist + prepare-package + + single + + + false + false + ${project.artifactId}-${project.version} + + hadoop-yarn-services-api + + + + + + + + rat diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties new file mode 100644 index 00000000000..333859e31e8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is the log4j configuration for Slider Application Master + +# Log rotation based on size (100KB) with a max of 10 backup files +log4j.rootLogger=INFO, amlog +log4j.threshhold=ALL +log4j.appender.amlog=org.apache.log4j.RollingFileAppender +log4j.appender.amlog.layout=org.apache.log4j.PatternLayout +log4j.appender.amlog.File=${LOG_DIR}/slider.log +log4j.appender.amlog.MaxFileSize=1MB +log4j.appender.amlog.MaxBackupIndex=10 + +# log layout skips stack-trace creation operations by avoiding line numbers and method +log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n + +# debug edition is much more expensive +#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n + +# configure stderr +# set the conversion pattern of stderr +# Print the date in ISO 8601 format +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.Target=System.err +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n + +log4j.appender.subprocess=org.apache.log4j.ConsoleAppender +log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout +log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n +#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess + +# for debugging Slider +#log4j.logger.org.apache.slider=DEBUG + +# uncomment to debug service lifecycle issues +#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG +#log4j.logger.org.apache.hadoop.yarn.service=DEBUG + +# uncomment for YARN operations +#log4j.logger.org.apache.hadoop.yarn.client=DEBUG + +# uncomment this to debug security problems +#log4j.logger.org.apache.hadoop.security=DEBUG + +#crank back on some noise +log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR +log4j.logger.org.apache.hadoop.hdfs=WARN +log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR + +log4j.logger.org.apache.zookeeper=WARN +log4j.logger.org.apache.curator.framework.state=ERROR +log4j.logger.org.apache.curator.framework.imps=WARN diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml index d778f447c35..a2c67c0add1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml @@ -331,6 +331,44 @@ + + dist + + false + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.hadoop + hadoop-assemblies + ${project.version} + + + + + dist + prepare-package + + single + + + false + false + ${project.artifactId}-${project.version} + + hadoop-yarn-slider-dist + + + + + + + + compile-protobuf diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 4b546cd5b41..d1f88c5fcf1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -2162,6 +2162,24 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME); hasServerLog4jProperties = log4jserver.isFile(); } + if (!hasServerLog4jProperties) { + // check for log4j properties in hadoop conf dir + String hadoopConfDir = System.getenv(ApplicationConstants.Environment + .HADOOP_CONF_DIR.name()); + if (hadoopConfDir != null) { + File localFile = new File(hadoopConfDir, SliderKeys + .LOG4J_SERVER_PROP_FILENAME); + if (localFile.exists()) { + Path localFilePath = createLocalPath(localFile); + remoteConfPath = new Path(clusterDirectory, + SliderKeys.SUBMITTED_CONF_DIR); + Path remoteFilePath = new Path(remoteConfPath, SliderKeys + .LOG4J_SERVER_PROP_FILENAME); + copy(config, localFilePath, remoteFilePath); + hasServerLog4jProperties = true; + } + } + } // the assumption here is that minimr cluster => this is a test run // and the classpath can look after itself @@ -2300,7 +2318,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // enable asserts commandLine.enableJavaAssertions(); - // if the conf dir has a log4j-server.properties, switch to that + // if the conf dir has a slideram-log4j.properties, switch to that if (hasServerLog4jProperties) { commandLine.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME); commandLine.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR); @@ -4471,14 +4489,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe return EXIT_SUCCESS; } - String libDir = System.getProperty(SliderKeys.PROPERTY_LIB_DIR); - if (isSet(libDir)) { - File srcFolder = new File(libDir); + String[] libDirs = SliderUtils.getLibDirs(); + if (libDirs.length > 0) { File tempLibTarGzipFile = File.createTempFile( SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME + "_", SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT); - // copy all jars except slider-core-.jar - tarGzipFolder(srcFolder, tempLibTarGzipFile, createJarFilter()); + // copy all jars + tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter()); log.info("Uploading dependency for AM (version {}) from {} to {}", version, tempLibTarGzipFile.toURI(), dependencyLibTarGzip.toUri()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 1484ee35439..3d25d331db3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -182,7 +182,7 @@ public interface SliderKeys extends SliderXmlConfKeys { /** * Slider AM log4j file name : {@value} */ - String LOG4J_SERVER_PROP_FILENAME = "log4j-server.properties"; + String LOG4J_SERVER_PROP_FILENAME = "slideram-log4j.properties"; /** * Standard log4j file name : {@value} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index f773982f18b..713cd02d86c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -25,6 +25,7 @@ import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -1399,6 +1400,22 @@ public final class SliderUtils { } } + public static String getLibDir() { + String[] libDirs = getLibDirs(); + if (libDirs == null || libDirs.length == 0) { + return null; + } + return libDirs[0]; + } + + public static String[] getLibDirs() { + String libDirStr = System.getProperty(SliderKeys.PROPERTY_LIB_DIR); + if (isUnset(libDirStr)) { + return ArrayUtils.EMPTY_STRING_ARRAY; + } + return StringUtils.split(libDirStr, ','); + } + /** * Submit a JAR containing a specific class and map it * @param providerResources provider map to build up @@ -1962,31 +1979,34 @@ public final class SliderUtils { /** * Given a source folder create a tar.gz file * - * @param srcFolder + * @param libDirs * @param tarGzipFile * * @throws IOException */ - public static void tarGzipFolder(File srcFolder, File tarGzipFile, + public static void tarGzipFolder(String[] libDirs, File tarGzipFile, FilenameFilter filter) throws IOException { - log.info("Tar-gzipping folder {} to {}", srcFolder.getAbsolutePath(), + log.info("Tar-gzipping folders {} to {}", libDirs, tarGzipFile.getAbsolutePath()); - List files = new ArrayList<>(); - generateFileList(files, srcFolder, srcFolder, true, filter); try(TarArchiveOutputStream taos = new TarArchiveOutputStream(new GZIPOutputStream( new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) { - for (String file : files) { - File srcFile = new File(srcFolder, file); - TarArchiveEntry tarEntry = new TarArchiveEntry( - srcFile, file); - taos.putArchiveEntry(tarEntry); - try(FileInputStream in = new FileInputStream(srcFile)) { - org.apache.commons.io.IOUtils.copy(in, taos); + for (String libDir : libDirs) { + File srcFolder = new File(libDir); + List files = new ArrayList<>(); + generateFileList(files, srcFolder, srcFolder, true, filter); + for (String file : files) { + File srcFile = new File(srcFolder, file); + TarArchiveEntry tarEntry = new TarArchiveEntry( + srcFile, file); + taos.putArchiveEntry(tarEntry); + try(FileInputStream in = new FileInputStream(srcFile)) { + org.apache.commons.io.IOUtils.copy(in, taos); + } + taos.flush(); + taos.closeArchiveEntry(); } - taos.flush(); - taos.closeArchiveEntry(); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java index fdc5be1b4f3..7ca469fbff4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java @@ -562,8 +562,7 @@ public class AgentClientProvider extends AbstractClientProvider } private void expandAgentTar(File agentPkgDir) throws IOException { - String libDirProp = - System.getProperty(PROPERTY_LIB_DIR); + String libDirProp = SliderUtils.getLibDir(); File tarFile = new File(libDirProp, AGENT_TAR); expandTar(tarFile, agentPkgDir); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java index e5430f28aa2..b58d3aaeaa3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java @@ -184,19 +184,20 @@ public class SliderAMClientProvider extends AbstractClientProvider libdir, miniClusterTestRun); - String libDirProp = - System.getProperty(SliderKeys.PROPERTY_LIB_DIR); log.info("Loading all dependencies for AM."); // If slider.tar.gz is available in hdfs use it, else upload all jars Path dependencyLibTarGzip = fileSystem.getDependencyTarGzip(); if (fileSystem.isFile(dependencyLibTarGzip)) { SliderUtils.putAmTarGzipAndUpdate(providerResources, fileSystem); } else { - ProviderUtils.addAllDependencyJars(providerResources, - fileSystem, - tempPath, - libdir, - libDirProp); + for (String libDirProp : SliderUtils.getLibDirs()) { + ProviderUtils.addAllDependencyJars(providerResources, + fileSystem, + tempPath, + libdir, + libDirProp); + + } } addKeytabResourceIfNecessary(fileSystem, instanceDescription, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java index 37061821271..56b84e087e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java @@ -82,16 +82,14 @@ public class TestPublishedConfigurationOutputter { String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)", ""); - assert output.contains( - "key1val1"); + assert output.contains("key1val1"); File file = tmpDir.newFile(); configurationOutputter.save(file); assert FileUtils.readFileToString(file, Charsets.UTF_8) .replaceAll("( |\\r|\\n)", "") - .contains( - "key1val1"); + .contains("key1val1"); } @Test @@ -103,14 +101,14 @@ public class TestPublishedConfigurationOutputter { String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)", ""); - assert output.contains("key1val1"); + assert output.contains("key1val1"); File file = tmpDir.newFile(); configurationOutputter.save(file); assert FileUtils.readFileToString(file, Charsets.UTF_8) .replaceAll("( |\\r|\\n)", "") - .contains( "key1val1"); + .contains("key1val1"); } @Test