MAPREDUCE-5814. fat jar with *-default.xml may fail when mapreduce.job.classloader=true. Contributed by Gera Shegalov
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1594326 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
c54a4bb666
commit
b0e80d1a47
|
@ -222,6 +222,9 @@ Release 2.5.0 - UNRELEASED
|
|||
MAPREDUCE-5888. Failed job leaves hung AM after it unregisters (Jason Lowe
|
||||
via jeagles)
|
||||
|
||||
MAPREDUCE-5814. fat jar with *-default.xml may fail when
|
||||
mapreduce.job.classloader=true. (Gera Shegalov via jlowe)
|
||||
|
||||
Release 2.4.1 - UNRELEASED
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
|
|
@ -33,6 +33,7 @@ import java.util.Map;
|
|||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
|
@ -344,8 +345,7 @@ public class MRApps extends Apps {
|
|||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("APP_CLASSPATH=" + appClasspath);
|
||||
}
|
||||
String[] systemClasses = conf.getStrings(
|
||||
MRJobConfig.MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES);
|
||||
String[] systemClasses = getSystemClasses(conf);
|
||||
ClassLoader jobClassLoader = createJobClassLoader(appClasspath,
|
||||
systemClasses);
|
||||
if (jobClassLoader != null) {
|
||||
|
@ -356,6 +356,12 @@ public class MRApps extends Apps {
|
|||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static String[] getSystemClasses(Configuration conf) {
|
||||
return conf.getTrimmedStrings(
|
||||
MRJobConfig.MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES);
|
||||
}
|
||||
|
||||
private static ClassLoader createJobClassLoader(final String appClasspath,
|
||||
final String[] systemClasses) throws IOException {
|
||||
try {
|
||||
|
|
|
@ -33,6 +33,7 @@ import java.io.IOException;
|
|||
import java.net.URI;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
|
@ -57,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
|
|||
import org.apache.hadoop.yarn.api.records.LocalResourceType;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||
import org.apache.hadoop.yarn.util.ApplicationClassLoader;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
@ -492,4 +494,36 @@ public class TestMRApps {
|
|||
assertTrue(MRApps.TaskStateUI.COMPLETED.correspondsTo(TaskState.KILLED));
|
||||
assertTrue(MRApps.TaskStateUI.RUNNING.correspondsTo(TaskState.RUNNING));
|
||||
}
|
||||
|
||||
|
||||
private static final String[] SYS_CLASSES = new String[] {
|
||||
"/java/fake/Klass",
|
||||
"/javax/fake/Klass",
|
||||
"/org/apache/commons/logging/fake/Klass",
|
||||
"/org/apache/log4j/fake/Klass",
|
||||
"/org/apache/hadoop/fake/Klass"
|
||||
};
|
||||
|
||||
private static final String[] DEFAULT_XMLS = new String[] {
|
||||
"core-default.xml",
|
||||
"mapred-default.xml",
|
||||
"hdfs-default.xml",
|
||||
"yarn-default.xml"
|
||||
};
|
||||
|
||||
@Test
|
||||
public void testSystemClasses() {
|
||||
final List<String> systemClasses =
|
||||
Arrays.asList(MRApps.getSystemClasses(new Configuration()));
|
||||
for (String defaultXml : DEFAULT_XMLS) {
|
||||
assertTrue(defaultXml + " must be system resource",
|
||||
ApplicationClassLoader.isSystemClass(defaultXml, systemClasses));
|
||||
}
|
||||
for (String klass : SYS_CLASSES) {
|
||||
assertTrue(klass + " must be system class",
|
||||
ApplicationClassLoader.isSystemClass(klass, systemClasses));
|
||||
}
|
||||
assertFalse("/fake/Klass must not be a system class",
|
||||
ApplicationClassLoader.isSystemClass("/fake/Klass", systemClasses));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1215,7 +1215,9 @@
|
|||
|
||||
<property>
|
||||
<name>mapreduce.job.classloader.system.classes</name>
|
||||
<value>java.,javax.,org.apache.commons.logging.,org.apache.log4j.,org.apache.hadoop.</value>
|
||||
<value>java.,javax.,org.apache.commons.logging.,org.apache.log4j.,
|
||||
org.apache.hadoop.,core-default.xml,hdfs-default.xml,
|
||||
mapred-default.xml,yarn-default.xml</value>
|
||||
<description>A comma-separated list of classes that should be loaded from the
|
||||
system classpath, not the user-supplied JARs, when mapreduce.job.classloader
|
||||
is enabled. Names ending in '.' (period) are treated as package names,
|
||||
|
|
|
@ -173,7 +173,7 @@ public class ApplicationClassLoader extends URLClassLoader {
|
|||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static boolean isSystemClass(String name, List<String> systemClasses) {
|
||||
public static boolean isSystemClass(String name, List<String> systemClasses) {
|
||||
if (systemClasses != null) {
|
||||
String canonicalName = name.replace('/', '.');
|
||||
while (canonicalName.startsWith(".")) {
|
||||
|
|
Loading…
Reference in New Issue