Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MAPREDUCE-6619. HADOOP_CLASSPATH is overwritten in MR container. Contributed by Junping Du

This commit is contained in:
Jian He 2016-01-27 13:16:36 -08:00
parent c25154576b
commit da18bbedaf
3 changed files with 62 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -409,6 +409,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6610. JobHistoryEventHandler should not swallow timeline response
(Li Lu via jianhe)
MAPREDUCE-6619. HADOOP_CLASSPATH is overwritten in MR container.
(Junping Du via jianhe)
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
View File

@ -261,6 +261,10 @@ public class MRApps extends Apps {
addClasspathToEnv(environment, classpathEnvVar, conf);
addClasspathToEnv(environment, hadoopClasspathEnvVar, conf);
// MAPREDUCE-6619, retain $HADOOP_CLASSPATH
MRApps.addToEnvironment(environment, hadoopClasspathEnvVar,
System.getenv(hadoopClasspathEnvVar), conf);
if (userClassesTakesPrecedence) {
MRApps.setMRFrameworkClasspath(environment, conf);
}

55
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
View File

@ -30,8 +30,10 @@ import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.lang.reflect.Field;
import java.net.URI;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -239,6 +241,12 @@ public class TestMRApps {
testTGZ.getAbsolutePath())).toString();
conf.set(MRJobConfig.CLASSPATH_ARCHIVES, testTGZQualifiedPath);
conf.set(MRJobConfig.CACHE_ARCHIVES, testTGZQualifiedPath + "#testTGZ");
// add hadoop.tgz to env HADOOP_CLASSPATH
Map<String, String> newEnv = new HashMap<String, String>();
newEnv.put(ApplicationConstants.Environment.HADOOP_CLASSPATH.name(),
"hadoop.tgz");
setEnv(newEnv);
Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, conf);
assertTrue(environment.get(ApplicationConstants.Environment.CLASSPATH.name()).startsWith(
@ -268,6 +276,53 @@ public class TestMRApps {
assertTrue(environment.get(
ApplicationConstants.Environment.HADOOP_CLASSPATH.name()).
contains("testTGZ"));
assertTrue(environment.get(
ApplicationConstants.Environment.HADOOP_CLASSPATH.name()).
contains("hadoop.tgz"));
}
// Only set env in runtime but not get persistent in OS (Linux or Windows)
@SuppressWarnings({"unchecked", "rawtypes"})
private static void setEnv(Map<String, String> newEnv) {
try {
// Hack for Windows
Class<?> processEnvClass =
Class.forName("java.lang.ProcessEnvironment");
Field theEnvField =
processEnvClass.getDeclaredField("theEnvironment");
theEnvField.setAccessible(true);
Map<String, String> currentEnv =
(Map<String, String>)theEnvField.get(null);
currentEnv.putAll(newEnv);
Field caseInsensitiveEnvField =
processEnvClass.getDeclaredField("theCaseInsensitiveEnvironment");
caseInsensitiveEnvField.setAccessible(true);
Map<String, String> ciEnv =
(Map<String, String>)caseInsensitiveEnvField.get(null);
ciEnv.putAll(newEnv);
} catch (NoSuchFieldException e) {
// Hack for Linux
try {
Class[] classes = Collections.class.getDeclaredClasses();
Map<String, String> env = System.getenv();
for (Class cl : classes) {
if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) {
Field field = cl.getDeclaredField("m");
field.setAccessible(true);
Object obj = field.get(env);
Map<String, String> map = (Map<String, String>) obj;
map.putAll(newEnv);
}
}
} catch (Exception e1) {
LOG.error("Hack env on Linux doesn't work:", e1);
throw new RuntimeException(e1);
}
} catch (Exception e) {
LOG.error("Hack env on Windows doesn't work:", e);
throw new RuntimeException(e);
}
}
@Test (timeout = 120000)