diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 08352a88c79..030e0dbf848 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -1234,11 +1234,15 @@ private String buildCommandLine(Service app, Configuration conf, return cmdStr; } - private Map addAMEnv() throws IOException { + @VisibleForTesting + protected Map addAMEnv() throws IOException { Map env = new HashMap<>(); - ClasspathConstructor classpath = - buildClasspath(YarnServiceConstants.SUBMITTED_CONF_DIR, "lib", fs, getConfig() - .getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)); + ClasspathConstructor classpath = buildClasspath( + YarnServiceConstants.SUBMITTED_CONF_DIR, + "lib", + fs, + getConfig().get(YarnServiceConf.YARN_SERVICE_CLASSPATH, ""), + getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)); env.put("CLASSPATH", classpath.buildClasspath()); env.put("LANG", "en_US.UTF-8"); env.put("LC_ALL", "en_US.UTF-8"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java index b9568f23fd1..58fe70b4f9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java @@ -60,6 +60,8 @@ public class YarnServiceConf { public static final String ROLLING_LOG_INCLUSION_PATTERN = "yarn.service.rolling-log.include-pattern"; public static final String ROLLING_LOG_EXCLUSION_PATTERN = "yarn.service.rolling-log.exclude-pattern"; + public static final String YARN_SERVICE_CLASSPATH = "yarn.service.classpath"; + public static final String YARN_SERVICES_SYSTEM_SERVICE_DIRECTORY = YARN_SERVICE_PREFIX + "system-service.dir"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java index a76b64d3cf0..34d2ba3bb83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java @@ -451,6 +451,7 @@ public static Path createLocalPath(File file) { * @param sliderConfDir relative path to the dir containing slider config * options to put on the classpath -or null * @param libdir directory containing the JAR files + * @param configClassPath extra class path configured in yarn-site.xml * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use * (and hence the current classpath should be used, not anything built up) * @return a classpath @@ -458,6 +459,7 @@ public static Path createLocalPath(File file) { public static ClasspathConstructor buildClasspath(String sliderConfDir, String libdir, SliderFileSystem sliderFileSystem, + String configClassPath, boolean usingMiniMRCluster) { ClasspathConstructor classpath = new ClasspathConstructor(); @@ -479,6 +481,11 @@ public static ClasspathConstructor buildClasspath(String sliderConfDir, classpath.addRemoteClasspathEnvVar(); classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$()); } + + if (!configClassPath.isEmpty()) { + classpath.appendAll(Arrays.asList(configClassPath.split(","))); + } + return classpath; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceClient.java index 4527da433ce..c66c4aedf89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceClient.java @@ -76,6 +76,29 @@ public class TestServiceClient { public ServiceTestUtils.ServiceFSWatcher rule = new ServiceTestUtils.ServiceFSWatcher(); + @Test + public void testAMEnvCustomClasspath() throws Exception { + Service service = createService(); + service.getComponents().forEach(comp -> + comp.setRestartPolicy(Component.RestartPolicyEnum.NEVER)); + ServiceClient client = MockServiceClient.create(rule, service, true); + //saving the original value of the param, for restoration purposes + String oldParam = client.getConfig().get("yarn.service.classpath", ""); + String originalPath = client.addAMEnv().get("CLASSPATH"); + + client.getConfig().set("yarn.service.classpath", "{{VAR_1}},{{VAR_2}}"); + String newPath = client.addAMEnv().get("CLASSPATH"); + + Assert.assertEquals(originalPath + "{{VAR_1}}{{VAR_2}}", newPath); + //restoring the original value for service classpath + client.getConfig().set("yarn.service.classpath", oldParam); + + newPath = client.addAMEnv().get("CLASSPATH"); + Assert.assertEquals(originalPath, newPath); + + client.stop(); + } + @Test public void testUpgradeDisabledByDefault() throws Exception { Service service = createService(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 9741f6c36b1..87c2f132ea7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -4211,4 +4211,14 @@ yarn.resourcemanager.activities-manager.app-activities.max-queue-length 1000 + + + + Comma separated extra class path parameters for yarn services AM. + These path elements will be appended to the end of the YARN service AM + classpath. + + yarn.service.classpath + +