diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 8401b6a151e..c64c65f11da 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -397,6 +397,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6610. JobHistoryEventHandler should not swallow timeline response (Li Lu via jianhe) + MAPREDUCE-6619. HADOOP_CLASSPATH is overwritten in MR container. + (Junping Du via jianhe) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index c6454654ee0..feea789efcc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -261,6 +261,10 @@ hadoopClasspathEnvVar, crossPlatformifyMREnv(conf, Environment.PWD), addClasspathToEnv(environment, classpathEnvVar, conf); addClasspathToEnv(environment, hadoopClasspathEnvVar, conf); + // MAPREDUCE-6619, retain $HADOOP_CLASSPATH + MRApps.addToEnvironment(environment, hadoopClasspathEnvVar, + System.getenv(hadoopClasspathEnvVar), conf); + if (userClassesTakesPrecedence) { MRApps.setMRFrameworkClasspath(environment, conf); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java index 3a417a0485e..f849d7219f9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java @@ -30,8 +30,10 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.lang.reflect.Field; import java.net.URI; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -239,6 +241,12 @@ public void testSetClasspathWithArchives () throws IOException { testTGZ.getAbsolutePath())).toString(); conf.set(MRJobConfig.CLASSPATH_ARCHIVES, testTGZQualifiedPath); conf.set(MRJobConfig.CACHE_ARCHIVES, testTGZQualifiedPath + "#testTGZ"); + // add hadoop.tgz to env HADOOP_CLASSPATH + Map newEnv = new HashMap(); + newEnv.put(ApplicationConstants.Environment.HADOOP_CLASSPATH.name(), + "hadoop.tgz"); + setEnv(newEnv); + Map environment = new HashMap(); MRApps.setClasspath(environment, conf); assertTrue(environment.get(ApplicationConstants.Environment.CLASSPATH.name()).startsWith( @@ -268,6 +276,53 @@ public void testSetClasspathWithArchives () throws IOException { assertTrue(environment.get( ApplicationConstants.Environment.HADOOP_CLASSPATH.name()). contains("testTGZ")); + assertTrue(environment.get( + ApplicationConstants.Environment.HADOOP_CLASSPATH.name()). + contains("hadoop.tgz")); + } + + // Only set env in runtime but not get persistent in OS (Linux or Windows) + @SuppressWarnings({"unchecked", "rawtypes"}) + private static void setEnv(Map newEnv) { + try { + // Hack for Windows + Class processEnvClass = + Class.forName("java.lang.ProcessEnvironment"); + Field theEnvField = + processEnvClass.getDeclaredField("theEnvironment"); + theEnvField.setAccessible(true); + Map currentEnv = + (Map)theEnvField.get(null); + currentEnv.putAll(newEnv); + + Field caseInsensitiveEnvField = + processEnvClass.getDeclaredField("theCaseInsensitiveEnvironment"); + caseInsensitiveEnvField.setAccessible(true); + Map ciEnv = + (Map)caseInsensitiveEnvField.get(null); + ciEnv.putAll(newEnv); + } catch (NoSuchFieldException e) { + // Hack for Linux + try { + Class[] classes = Collections.class.getDeclaredClasses(); + Map env = System.getenv(); + for (Class cl : classes) { + if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) { + Field field = cl.getDeclaredField("m"); + field.setAccessible(true); + Object obj = field.get(env); + Map map = (Map) obj; + map.putAll(newEnv); + } + } + } catch (Exception e1) { + LOG.error("Hack env on Linux doesn't work:", e1); + throw new RuntimeException(e1); + } + } catch (Exception e) { + LOG.error("Hack env on Windows doesn't work:", e); + throw new RuntimeException(e); + } } @Test (timeout = 120000)