From 0fb55f1aecc81990e1fc0b2f9645611e8f875d5b Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 28 Aug 2013 05:14:15 +0000 Subject: [PATCH] YARN-602. Fixed NodeManager to not let users override some mandatory environmental variables. Contributed by Kenji Kikushima. svn merge --ignore-ancestry -c 1518077 ../../trunk/ git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1518078 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../launcher/ContainerLaunch.java | 12 ++-- .../launcher/TestContainerLaunch.java | 59 ++++++++++++++++++- 3 files changed, 65 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e1b4749806c..07134caa3d3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -96,6 +96,9 @@ Release 2.1.1-beta - UNRELEASED YARN-981. Fixed YARN webapp so that /logs servlet works like before. (Jian He via vinodkv) + YARN-602. Fixed NodeManager to not let users override some mandatory + environmental variables. (Kenji Kikushima via vinodkv) + Release 2.1.0-beta - 2013-08-22 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index f4578dc83d8..89812d27efc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -588,20 +588,18 @@ public class ContainerLaunch implements Callable { environment.put(Environment.LOG_DIRS.name(), StringUtils.join(",", containerLogDirs)); - putEnvIfNotNull(environment, Environment.USER.name(), container.getUser()); + environment.put(Environment.USER.name(), container.getUser()); - putEnvIfNotNull(environment, - Environment.LOGNAME.name(),container.getUser()); - - putEnvIfNotNull(environment, - Environment.HOME.name(), + environment.put(Environment.LOGNAME.name(), container.getUser()); + + environment.put(Environment.HOME.name(), conf.get( YarnConfiguration.NM_USER_HOME_DIR, YarnConfiguration.DEFAULT_NM_USER_HOME_DIR ) ); - putEnvIfNotNull(environment, Environment.PWD.name(), pwd.toString()); + environment.put(Environment.PWD.name(), pwd.toString()); putEnvIfNotNull(environment, Environment.HADOOP_CONF_DIR.name(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 78c78c0d605..d25c1c4039b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -346,7 +346,6 @@ public class TestContainerLaunch extends BaseContainerManagerTest { ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); - int port = 12345; ContainerId cId = ContainerId.newInstance(appAttemptId, 0); Map userSetEnv = new HashMap(); userSetEnv.put(Environment.CONTAINER_ID.name(), "user_set_container_id"); @@ -354,6 +353,11 @@ public class TestContainerLaunch extends BaseContainerManagerTest { userSetEnv.put(Environment.NM_PORT.name(), "user_set_NM_PORT"); userSetEnv.put(Environment.NM_HTTP_PORT.name(), "user_set_NM_HTTP_PORT"); userSetEnv.put(Environment.LOCAL_DIRS.name(), "user_set_LOCAL_DIR"); + userSetEnv.put(Environment.USER.key(), "user_set_" + + Environment.USER.key()); + userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME"); + userSetEnv.put(Environment.PWD.name(), "user_set_PWD"); + userSetEnv.put(Environment.HOME.name(), "user_set_HOME"); containerLaunchContext.setEnvironment(userSetEnv); File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile"); @@ -371,6 +375,14 @@ public class TestContainerLaunch extends BaseContainerManagerTest { + processStartFile); fileWriter.println("@echo " + Environment.LOCAL_DIRS.$() + ">> " + processStartFile); + fileWriter.println("@echo " + Environment.USER.$() + ">> " + + processStartFile); + fileWriter.println("@echo " + Environment.LOGNAME.$() + ">> " + + processStartFile); + fileWriter.println("@echo " + Environment.PWD.$() + ">> " + + processStartFile); + fileWriter.println("@echo " + Environment.HOME.$() + ">> " + + processStartFile); fileWriter.println("@echo " + cId + ">> " + processStartFile); fileWriter.println("@ping -n 100 127.0.0.1 >nul"); } else { @@ -385,6 +397,14 @@ public class TestContainerLaunch extends BaseContainerManagerTest { + processStartFile); fileWriter.write("\necho $" + Environment.LOCAL_DIRS.name() + " >> " + processStartFile); + fileWriter.write("\necho $" + Environment.USER.name() + " >> " + + processStartFile); + fileWriter.write("\necho $" + Environment.LOGNAME.name() + " >> " + + processStartFile); + fileWriter.write("\necho $" + Environment.PWD.name() + " >> " + + processStartFile); + fileWriter.write("\necho $" + Environment.HOME.name() + " >> " + + processStartFile); fileWriter.write("\necho $$ >> " + processStartFile); fileWriter.write("\nexec sleep 100"); } @@ -452,6 +472,22 @@ public class TestContainerLaunch extends BaseContainerManagerTest { reader.readLine()); Assert.assertEquals(String.valueOf(HTTP_PORT), reader.readLine()); Assert.assertEquals(StringUtils.join(",", appDirs), reader.readLine()); + Assert.assertEquals(user, reader.readLine()); + Assert.assertEquals(user, reader.readLine()); + String obtainedPWD = reader.readLine(); + boolean found = false; + for (Path localDir : appDirs) { + if (new Path(localDir, cId.toString()).toString().equals(obtainedPWD)) { + found = true; + break; + } + } + Assert.assertTrue("Wrong local-dir found : " + obtainedPWD, found); + Assert.assertEquals( + conf.get( + YarnConfiguration.NM_USER_HOME_DIR, + YarnConfiguration.DEFAULT_NM_USER_HOME_DIR), + reader.readLine()); Assert.assertEquals(cId.toString(), containerLaunchContext .getEnvironment().get(Environment.CONTAINER_ID.name())); @@ -465,6 +501,26 @@ public class TestContainerLaunch extends BaseContainerManagerTest { .getEnvironment().get(Environment.LOCAL_DIRS.name())); Assert.assertEquals(StringUtils.join(",", containerLogDirs), containerLaunchContext.getEnvironment().get(Environment.LOG_DIRS.name())); + Assert.assertEquals(user, containerLaunchContext.getEnvironment() + .get(Environment.USER.name())); + Assert.assertEquals(user, containerLaunchContext.getEnvironment() + .get(Environment.LOGNAME.name())); + found = false; + obtainedPWD = + containerLaunchContext.getEnvironment().get(Environment.PWD.name()); + for (Path localDir : appDirs) { + if (new Path(localDir, cId.toString()).toString().equals(obtainedPWD)) { + found = true; + break; + } + } + Assert.assertTrue("Wrong local-dir found : " + obtainedPWD, found); + Assert.assertEquals( + conf.get( + YarnConfiguration.NM_USER_HOME_DIR, + YarnConfiguration.DEFAULT_NM_USER_HOME_DIR), + containerLaunchContext.getEnvironment() + .get(Environment.HOME.name())); // Get the pid of the process String pid = reader.readLine().trim(); @@ -538,7 +594,6 @@ public class TestContainerLaunch extends BaseContainerManagerTest { ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); - int port = 12345; // upload the script file so that the container can run it URL resource_alpha =