From fb3d53e001725ab6c0182ba3be08e1b6342f9822 Mon Sep 17 00:00:00 2001 From: Eric Payne Date: Wed, 13 Sep 2017 14:41:55 -0500 Subject: [PATCH] YARN-4727. Unable to override the $HADOOP_CONF_DIR env variable for container. Contributed by Jason Lowe. (cherry picked from commit 3860be7961580ac20dd505d665b580f0a04ac4f8) --- .../launcher/ContainerLaunch.java | 5 +-- .../launcher/TestContainerLaunch.java | 32 +++++++++++++------ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index d29afd8b949..88f5925bd30 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -1124,10 +1124,7 @@ public class ContainerLaunch implements Callable { environment.put(Environment.PWD.name(), pwd.toString()); - putEnvIfNotNull(environment, - Environment.HADOOP_CONF_DIR.name(), - System.getenv(Environment.HADOOP_CONF_DIR.name()) - ); + putEnvIfAbsent(environment, Environment.HADOOP_CONF_DIR.name()); if (!Shell.WINDOWS) { environment.put("JVM_PID", "$$"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index ea46c29e825..c75ee04f5a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -47,6 +47,7 @@ import java.util.StringTokenizer; import java.util.jar.JarFile; import java.util.jar.Manifest; +import com.google.common.base.Supplier; import com.google.common.collect.Lists; import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; @@ -55,6 +56,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.SecretManager.InvalidToken; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ExitCodeException; import org.apache.hadoop.util.StringUtils; @@ -691,11 +693,15 @@ public class TestContainerLaunch extends BaseContainerManagerTest { userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME"); userSetEnv.put(Environment.PWD.name(), "user_set_PWD"); userSetEnv.put(Environment.HOME.name(), "user_set_HOME"); + final String userConfDir = "user_set_HADOOP_CONF_DIR"; + userSetEnv.put(Environment.HADOOP_CONF_DIR.name(), userConfDir); containerLaunchContext.setEnvironment(userSetEnv); File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile"); PrintWriter fileWriter = new PrintWriter(scriptFile); File processStartFile = + new File(tmpDir, "env_vars.tmp").getAbsoluteFile(); + final File processFinalFile = new File(tmpDir, "env_vars.txt").getAbsoluteFile(); if (Shell.WINDOWS) { fileWriter.println("@echo " + Environment.CONTAINER_ID.$() + "> " @@ -716,6 +722,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest { + processStartFile); fileWriter.println("@echo " + Environment.HOME.$() + ">> " + processStartFile); + fileWriter.println("@echo " + Environment.HADOOP_CONF_DIR.$() + ">> " + + processStartFile); for (String serviceName : containerManager.getAuxServiceMetaData() .keySet()) { fileWriter.println("@echo %" + AuxiliaryServiceHelper.NM_AUX_SERVICE @@ -723,6 +731,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest { + processStartFile); } fileWriter.println("@echo " + cId + ">> " + processStartFile); + fileWriter.println("@move /Y " + processStartFile + " " + + processFinalFile); fileWriter.println("@ping -n 100 127.0.0.1 >nul"); } else { fileWriter.write("\numask 0"); // So that start file is readable by the test @@ -744,6 +754,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest { + processStartFile); fileWriter.write("\necho $" + Environment.HOME.name() + " >> " + processStartFile); + fileWriter.write("\necho $" + Environment.HADOOP_CONF_DIR.name() + " >> " + + processStartFile); for (String serviceName : containerManager.getAuxServiceMetaData() .keySet()) { fileWriter.write("\necho $" + AuxiliaryServiceHelper.NM_AUX_SERVICE @@ -751,6 +763,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest { + processStartFile); } fileWriter.write("\necho $$ >> " + processStartFile); + fileWriter.write("\nmv " + processStartFile + " " + processFinalFile); fileWriter.write("\nexec sleep 100"); } fileWriter.close(); @@ -784,13 +797,12 @@ public class TestContainerLaunch extends BaseContainerManagerTest { StartContainersRequest.newInstance(list); containerManager.startContainers(allRequests); - int timeoutSecs = 0; - while (!processStartFile.exists() && timeoutSecs++ < 20) { - Thread.sleep(1000); - LOG.info("Waiting for process start-file to be created"); - } - Assert.assertTrue("ProcessStartFile doesn't exist!", - processStartFile.exists()); + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return processFinalFile.exists(); + } + }, 10, 20000); // Now verify the contents of the file List localDirs = dirsHandler.getLocalDirs(); @@ -810,7 +822,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest { containerLogDirs.add(logDir + Path.SEPARATOR + relativeContainerLogDir); } BufferedReader reader = - new BufferedReader(new FileReader(processStartFile)); + new BufferedReader(new FileReader(processFinalFile)); Assert.assertEquals(cId.toString(), reader.readLine()); Assert.assertEquals(context.getNodeId().getHost(), reader.readLine()); Assert.assertEquals(String.valueOf(context.getNodeId().getPort()), @@ -833,7 +845,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest { YarnConfiguration.NM_USER_HOME_DIR, YarnConfiguration.DEFAULT_NM_USER_HOME_DIR), reader.readLine()); - + Assert.assertEquals(userConfDir, reader.readLine()); for (String serviceName : containerManager.getAuxServiceMetaData().keySet()) { Assert.assertEquals( containerManager.getAuxServiceMetaData().get(serviceName), @@ -872,6 +884,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest { YarnConfiguration.DEFAULT_NM_USER_HOME_DIR), containerLaunchContext.getEnvironment() .get(Environment.HOME.name())); + Assert.assertEquals(userConfDir, containerLaunchContext.getEnvironment() + .get(Environment.HADOOP_CONF_DIR.name())); // Get the pid of the process String pid = reader.readLine().trim();