From 3860be7961580ac20dd505d665b580f0a04ac4f8 Mon Sep 17 00:00:00 2001 From: Eric Payne Date: Wed, 13 Sep 2017 14:41:55 -0500 Subject: [PATCH] YARN-4727. Unable to override the $HADOOP_CONF_DIR env variable for container. Contributed by Jason Lowe. --- .../launcher/ContainerLaunch.java | 5 +-- .../launcher/TestContainerLaunch.java | 32 +++++++++++++------ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 50443f31fdf..d0ce787aaa6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -1148,10 +1148,7 @@ public void sanitizeEnv(Map environment, Path pwd, environment.put(Environment.PWD.name(), pwd.toString()); - putEnvIfNotNull(environment, - Environment.HADOOP_CONF_DIR.name(), - System.getenv(Environment.HADOOP_CONF_DIR.name()) - ); + putEnvIfAbsent(environment, Environment.HADOOP_CONF_DIR.name()); if (!Shell.WINDOWS) { environment.put("JVM_PID", "$$"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 085b60f9a32..7176942303d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -46,6 +46,7 @@ import java.util.jar.JarFile; import java.util.jar.Manifest; +import com.google.common.base.Supplier; import com.google.common.collect.Lists; import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; @@ -54,6 +55,7 @@ import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.SecretManager.InvalidToken; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ExitCodeException; import org.apache.hadoop.util.StringUtils; @@ -724,11 +726,15 @@ public void testContainerEnvVariables() throws Exception { userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME"); userSetEnv.put(Environment.PWD.name(), "user_set_PWD"); userSetEnv.put(Environment.HOME.name(), "user_set_HOME"); + final String userConfDir = "user_set_HADOOP_CONF_DIR"; + userSetEnv.put(Environment.HADOOP_CONF_DIR.name(), userConfDir); containerLaunchContext.setEnvironment(userSetEnv); File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile"); PrintWriter fileWriter = new PrintWriter(scriptFile); File processStartFile = + new File(tmpDir, "env_vars.tmp").getAbsoluteFile(); + final File processFinalFile = new File(tmpDir, "env_vars.txt").getAbsoluteFile(); if (Shell.WINDOWS) { fileWriter.println("@echo " + Environment.CONTAINER_ID.$() + "> " @@ -749,6 +755,8 @@ public void testContainerEnvVariables() throws Exception { + processStartFile); fileWriter.println("@echo " + Environment.HOME.$() + ">> " + processStartFile); + fileWriter.println("@echo " + Environment.HADOOP_CONF_DIR.$() + ">> " + + processStartFile); for (String serviceName : containerManager.getAuxServiceMetaData() .keySet()) { fileWriter.println("@echo %" + AuxiliaryServiceHelper.NM_AUX_SERVICE @@ -756,6 +764,8 @@ public void testContainerEnvVariables() throws Exception { + processStartFile); } fileWriter.println("@echo " + cId + ">> " + processStartFile); + fileWriter.println("@move /Y " + processStartFile + " " + + processFinalFile); fileWriter.println("@ping -n 100 127.0.0.1 >nul"); } else { fileWriter.write("\numask 0"); // So that start file is readable by the test @@ -777,6 +787,8 @@ public void testContainerEnvVariables() throws Exception { + processStartFile); fileWriter.write("\necho $" + Environment.HOME.name() + " >> " + processStartFile); + fileWriter.write("\necho $" + Environment.HADOOP_CONF_DIR.name() + " >> " + + processStartFile); for (String serviceName : containerManager.getAuxServiceMetaData() .keySet()) { fileWriter.write("\necho $" + AuxiliaryServiceHelper.NM_AUX_SERVICE @@ -784,6 +796,7 @@ public void testContainerEnvVariables() throws Exception { + processStartFile); } fileWriter.write("\necho $$ >> " + processStartFile); + fileWriter.write("\nmv " + processStartFile + " " + processFinalFile); fileWriter.write("\nexec sleep 100"); } fileWriter.close(); @@ -817,13 +830,12 @@ public void testContainerEnvVariables() throws Exception { StartContainersRequest.newInstance(list); containerManager.startContainers(allRequests); - int timeoutSecs = 0; - while (!processStartFile.exists() && timeoutSecs++ < 20) { - Thread.sleep(1000); - LOG.info("Waiting for process start-file to be created"); - } - Assert.assertTrue("ProcessStartFile doesn't exist!", - processStartFile.exists()); + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return processFinalFile.exists(); + } + }, 10, 20000); // Now verify the contents of the file List localDirs = dirsHandler.getLocalDirs(); @@ -843,7 +855,7 @@ public void testContainerEnvVariables() throws Exception { containerLogDirs.add(logDir + Path.SEPARATOR + relativeContainerLogDir); } BufferedReader reader = - new BufferedReader(new FileReader(processStartFile)); + new BufferedReader(new FileReader(processFinalFile)); Assert.assertEquals(cId.toString(), reader.readLine()); Assert.assertEquals(context.getNodeId().getHost(), reader.readLine()); Assert.assertEquals(String.valueOf(context.getNodeId().getPort()), @@ -866,7 +878,7 @@ public void testContainerEnvVariables() throws Exception { YarnConfiguration.NM_USER_HOME_DIR, YarnConfiguration.DEFAULT_NM_USER_HOME_DIR), reader.readLine()); - + Assert.assertEquals(userConfDir, reader.readLine()); for (String serviceName : containerManager.getAuxServiceMetaData().keySet()) { Assert.assertEquals( containerManager.getAuxServiceMetaData().get(serviceName), @@ -905,6 +917,8 @@ public void testContainerEnvVariables() throws Exception { YarnConfiguration.DEFAULT_NM_USER_HOME_DIR), containerLaunchContext.getEnvironment() .get(Environment.HOME.name())); + Assert.assertEquals(userConfDir, containerLaunchContext.getEnvironment() + .get(Environment.HADOOP_CONF_DIR.name())); // Get the pid of the process String pid = reader.readLine().trim();