From 890cd33bf76519dfa0df9ba197a344b52005fb5e Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Tue, 13 May 2014 18:59:56 +0000 Subject: [PATCH] svn merge -c 1594326 FIXES: MAPREDUCE-5814. fat jar with *-default.xml may fail when mapreduce.job.classloader=true. Contributed by Gera Shegalov git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1594328 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../hadoop/mapreduce/v2/util/MRApps.java | 10 ++++-- .../hadoop/mapreduce/v2/util/TestMRApps.java | 34 +++++++++++++++++++ .../src/main/resources/mapred-default.xml | 4 ++- .../yarn/util/ApplicationClassLoader.java | 2 +- 5 files changed, 49 insertions(+), 4 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 08a6d248b82..6e1c79386e4 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -83,6 +83,9 @@ Release 2.5.0 - UNRELEASED MAPREDUCE-5888. Failed job leaves hung AM after it unregisters (Jason Lowe via jeagles) + MAPREDUCE-5814. fat jar with *-default.xml may fail when + mapreduce.job.classloader=true. (Gera Shegalov via jlowe) + Release 2.4.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index e3294d70fae..d72a1bff269 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -344,8 +345,7 @@ public class MRApps extends Apps { if (LOG.isDebugEnabled()) { LOG.debug("APP_CLASSPATH=" + appClasspath); } - String[] systemClasses = conf.getStrings( - MRJobConfig.MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES); + String[] systemClasses = getSystemClasses(conf); ClassLoader jobClassLoader = createJobClassLoader(appClasspath, systemClasses); if (jobClassLoader != null) { @@ -356,6 +356,12 @@ public class MRApps extends Apps { } } + @VisibleForTesting + static String[] getSystemClasses(Configuration conf) { + return conf.getTrimmedStrings( + MRJobConfig.MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES); + } + private static ClassLoader createJobClassLoader(final String appClasspath, final String[] systemClasses) throws IOException { try { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java index fd59b8170d4..36a6a2a9c6f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.net.URI; import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; @@ -58,6 +59,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.util.ApplicationClassLoader; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -488,4 +490,36 @@ public class TestMRApps { assertTrue(MRApps.TaskStateUI.COMPLETED.correspondsTo(TaskState.KILLED)); assertTrue(MRApps.TaskStateUI.RUNNING.correspondsTo(TaskState.RUNNING)); } + + + private static final String[] SYS_CLASSES = new String[] { + "/java/fake/Klass", + "/javax/fake/Klass", + "/org/apache/commons/logging/fake/Klass", + "/org/apache/log4j/fake/Klass", + "/org/apache/hadoop/fake/Klass" + }; + + private static final String[] DEFAULT_XMLS = new String[] { + "core-default.xml", + "mapred-default.xml", + "hdfs-default.xml", + "yarn-default.xml" + }; + + @Test + public void testSystemClasses() { + final List systemClasses = + Arrays.asList(MRApps.getSystemClasses(new Configuration())); + for (String defaultXml : DEFAULT_XMLS) { + assertTrue(defaultXml + " must be system resource", + ApplicationClassLoader.isSystemClass(defaultXml, systemClasses)); + } + for (String klass : SYS_CLASSES) { + assertTrue(klass + " must be system class", + ApplicationClassLoader.isSystemClass(klass, systemClasses)); + } + assertFalse("/fake/Klass must not be a system class", + ApplicationClassLoader.isSystemClass("/fake/Klass", systemClasses)); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index fa6741953bd..1a25554b45d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1648,7 +1648,9 @@ mapreduce.job.classloader.system.classes - java.,javax.,org.apache.commons.logging.,org.apache.log4j.,org.apache.hadoop. + java.,javax.,org.apache.commons.logging.,org.apache.log4j., + org.apache.hadoop.,core-default.xml,hdfs-default.xml, + mapred-default.xml,yarn-default.xml A comma-separated list of classes that should be loaded from the system classpath, not the user-supplied JARs, when mapreduce.job.classloader is enabled. Names ending in '.' (period) are treated as package names, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java index 6dcea6ca05c..63dc5b798c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java @@ -173,7 +173,7 @@ public class ApplicationClassLoader extends URLClassLoader { } @VisibleForTesting - static boolean isSystemClass(String name, List systemClasses) { + public static boolean isSystemClass(String name, List systemClasses) { if (systemClasses != null) { String canonicalName = name.replace('/', '.'); while (canonicalName.startsWith(".")) {