YARN-9546. Add configuration option for YARN Native services AM classpath. Contributed by Gergely Pollak.
This commit is contained in:
parent
0d1d7c86ec
commit
24c53e057a
|
@ -1234,11 +1234,15 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
|
|||
return cmdStr;
|
||||
}
|
||||
|
||||
private Map<String, String> addAMEnv() throws IOException {
|
||||
@VisibleForTesting
|
||||
protected Map<String, String> addAMEnv() throws IOException {
|
||||
Map<String, String> env = new HashMap<>();
|
||||
ClasspathConstructor classpath =
|
||||
buildClasspath(YarnServiceConstants.SUBMITTED_CONF_DIR, "lib", fs, getConfig()
|
||||
.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false));
|
||||
ClasspathConstructor classpath = buildClasspath(
|
||||
YarnServiceConstants.SUBMITTED_CONF_DIR,
|
||||
"lib",
|
||||
fs,
|
||||
getConfig().get(YarnServiceConf.YARN_SERVICE_CLASSPATH, ""),
|
||||
getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false));
|
||||
env.put("CLASSPATH", classpath.buildClasspath());
|
||||
env.put("LANG", "en_US.UTF-8");
|
||||
env.put("LC_ALL", "en_US.UTF-8");
|
||||
|
|
|
@ -60,6 +60,8 @@ public class YarnServiceConf {
|
|||
public static final String ROLLING_LOG_INCLUSION_PATTERN = "yarn.service.rolling-log.include-pattern";
|
||||
public static final String ROLLING_LOG_EXCLUSION_PATTERN = "yarn.service.rolling-log.exclude-pattern";
|
||||
|
||||
public static final String YARN_SERVICE_CLASSPATH = "yarn.service.classpath";
|
||||
|
||||
public static final String YARN_SERVICES_SYSTEM_SERVICE_DIRECTORY =
|
||||
YARN_SERVICE_PREFIX + "system-service.dir";
|
||||
|
||||
|
|
|
@ -451,6 +451,7 @@ public final class ServiceUtils {
|
|||
* @param sliderConfDir relative path to the dir containing slider config
|
||||
* options to put on the classpath -or null
|
||||
* @param libdir directory containing the JAR files
|
||||
* @param configClassPath extra class path configured in yarn-site.xml
|
||||
* @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
|
||||
* (and hence the current classpath should be used, not anything built up)
|
||||
* @return a classpath
|
||||
|
@ -458,6 +459,7 @@ public final class ServiceUtils {
|
|||
public static ClasspathConstructor buildClasspath(String sliderConfDir,
|
||||
String libdir,
|
||||
SliderFileSystem sliderFileSystem,
|
||||
String configClassPath,
|
||||
boolean usingMiniMRCluster) {
|
||||
|
||||
ClasspathConstructor classpath = new ClasspathConstructor();
|
||||
|
@ -479,6 +481,11 @@ public final class ServiceUtils {
|
|||
classpath.addRemoteClasspathEnvVar();
|
||||
classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
|
||||
}
|
||||
|
||||
if (!configClassPath.isEmpty()) {
|
||||
classpath.appendAll(Arrays.asList(configClassPath.split(",")));
|
||||
}
|
||||
|
||||
return classpath;
|
||||
}
|
||||
|
||||
|
|
|
@ -76,6 +76,29 @@ public class TestServiceClient {
|
|||
public ServiceTestUtils.ServiceFSWatcher rule =
|
||||
new ServiceTestUtils.ServiceFSWatcher();
|
||||
|
||||
@Test
|
||||
public void testAMEnvCustomClasspath() throws Exception {
|
||||
Service service = createService();
|
||||
service.getComponents().forEach(comp ->
|
||||
comp.setRestartPolicy(Component.RestartPolicyEnum.NEVER));
|
||||
ServiceClient client = MockServiceClient.create(rule, service, true);
|
||||
//saving the original value of the param, for restoration purposes
|
||||
String oldParam = client.getConfig().get("yarn.service.classpath", "");
|
||||
String originalPath = client.addAMEnv().get("CLASSPATH");
|
||||
|
||||
client.getConfig().set("yarn.service.classpath", "{{VAR_1}},{{VAR_2}}");
|
||||
String newPath = client.addAMEnv().get("CLASSPATH");
|
||||
|
||||
Assert.assertEquals(originalPath + "<CPS>{{VAR_1}}<CPS>{{VAR_2}}", newPath);
|
||||
//restoring the original value for service classpath
|
||||
client.getConfig().set("yarn.service.classpath", oldParam);
|
||||
|
||||
newPath = client.addAMEnv().get("CLASSPATH");
|
||||
Assert.assertEquals(originalPath, newPath);
|
||||
|
||||
client.stop();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUpgradeDisabledByDefault() throws Exception {
|
||||
Service service = createService();
|
||||
|
|
|
@ -4211,4 +4211,14 @@
|
|||
<name>yarn.resourcemanager.activities-manager.app-activities.max-queue-length</name>
|
||||
<value>1000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>
|
||||
Comma separated extra class path parameters for yarn services AM.
|
||||
These path elements will be appended to the end of the YARN service AM
|
||||
classpath.
|
||||
</description>
|
||||
<name>yarn.service.classpath</name>
|
||||
<value></value>
|
||||
</property>
|
||||
</configuration>
|
||||
|
|
Loading…
Reference in New Issue