Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-10893. isolated classloader on the client side. Contributed by Sangjin Lee 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1619604 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jason Darrell Lowe 2014-08-21 21:38:16 +00:00
parent cbbb899aed
commit da4ba50269
20 changed files with 584 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -383,6 +383,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10433. Key Management Server based on KeyProvider API. (tucu)
HADOOP-10893. isolated classloader on the client side (Sangjin Lee via
jlowe)
IMPROVEMENTS
HADOOP-10808. Remove unused native code for munlock. (cnauroth)

5
hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
View File

@ -108,6 +108,11 @@
<Method name="driver" />
<Bug pattern="DM_EXIT" />
</Match>
<Match>
<Class name="org.apache.hadoop.util.RunJar" />
<Method name="run" />
<Bug pattern="DM_EXIT" />
</Match>
<!--
We need to cast objects between old and new api objects
-->

10
hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd
View File

@ -282,10 +282,12 @@ if not "%HADOOP_MAPRED_HOME%\%MAPRED_DIR%" == "%HADOOP_YARN_HOME%\%YARN_DIR%" (
@rem
if defined HADOOP_CLASSPATH (
if defined HADOOP_USER_CLASSPATH_FIRST (
set CLASSPATH=%HADOOP_CLASSPATH%;%CLASSPATH%;
) else (
set CLASSPATH=%CLASSPATH%;%HADOOP_CLASSPATH%;
if not defined HADOOP_USE_CLIENT_CLASSLOADER (
if defined HADOOP_USER_CLASSPATH_FIRST (
set CLASSPATH=%HADOOP_CLASSPATH%;%CLASSPATH%;
) else (
set CLASSPATH=%CLASSPATH%;%HADOOP_CLASSPATH%;
)
)
)

21
hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
View File

@ -450,7 +450,8 @@ function hadoop_add_to_classpath_mapred
function hadoop_add_to_classpath_userpath
{
# Add the user-specified HADOOP_CLASSPATH to the
# official CLASSPATH env var.
# official CLASSPATH env var if HADOOP_USE_CLIENT_CLASSLOADER
# is not set.
# Add it first or last depending on if user has
# set env-var HADOOP_USER_CLASSPATH_FIRST
# we'll also dedupe it, because we're cool like that.
@ -469,14 +470,16 @@ function hadoop_add_to_classpath_userpath
done
let j=c-1
if [[ -z "${HADOOP_USER_CLASSPATH_FIRST}" ]]; then
for ((i=j; i>=0; i--)); do
hadoop_add_classpath "${array[$i]}" before
done
else
for ((i=0; i<=j; i++)); do
hadoop_add_classpath "${array[$i]}" after
done
if [[ -z "${HADOOP_USE_CLIENT_CLASSLOADER}" ]]; then
if [[ -z "${HADOOP_USER_CLASSPATH_FIRST}" ]]; then
for ((i=j; i>=0; i--)); do
hadoop_add_classpath "${array[$i]}" before
done
else
for ((i=0; i<=j; i++)); do
hadoop_add_classpath "${array[$i]}" after
done
fi
fi
fi
}

20
hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
View File

@ -28,6 +28,26 @@
@rem classpath. Can be defined, for example,
@rem by doing
@rem export HADOOP_USER_CLASSPATH_FIRST=true
@rem
@rem HADOOP_USE_CLIENT_CLASSLOADER When defined, HADOOP_CLASSPATH and the
@rem jar as the hadoop jar argument are
@rem handled by a separate isolated client
@rem classloader. If it is set,
@rem HADOOP_USER_CLASSPATH_FIRST is
@rem ignored. Can be defined by doing
@rem export HADOOP_USE_CLIENT_CLASSLOADER=true
@rem
@rem HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES
@rem When defined, it overrides the default
@rem definition of system classes for the
@rem client classloader when
@rem HADOOP_USE_CLIENT_CLASSLOADER is
@rem enabled. Names ending in '.' (period)
@rem are treated as package names, and names
@rem starting with a '-' are treated as
@rem negative matches. For example,
@rem export HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES="-org.apache.hadoop.UserClass,java.,javax.,org.apache.hadoop."
@rem
@rem HADOOP_HEAPSIZE The maximum amount of heap to use, in MB.
@rem Default is 1000.

11
hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
View File

@ -111,6 +111,17 @@ esac
# Should HADOOP_USER_CLASSPATH be first in the official CLASSPATH?
# export HADOOP_USER_CLASSPATH_FIRST="yes"
# If HADOOP_USE_CLIENT_CLASSLOADER is set, HADOOP_CLASSPATH along with the main
# jar are handled by a separate isolated client classloader. If it is set,
# HADOOP_USER_CLASSPATH_FIRST is ignored. Can be defined by doing
# export HADOOP_USE_CLIENT_CLASSLOADER=true
# HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES overrides the default definition of
# system classes for the client classloader when HADOOP_USE_CLIENT_CLASSLOADER
# is enabled. Names ending in '.' (period) are treated as package names, and
# names starting with a '-' are treated as negative matches. For example,
# export HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES="-org.apache.hadoop.UserClass,java.,javax.,org.apache.hadoop."
###
# Options for remote shell connectivity
###

219
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java Normal file
View File

@ -0,0 +1,219 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.util;
import java.io.File;
import java.io.FilenameFilter;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
/**
* A {@link URLClassLoader} for application isolation. Classes from the
* application JARs are loaded in preference to the parent loader.
*/
@Public
@Unstable
public class ApplicationClassLoader extends URLClassLoader {
/**
* Default value of the system classes if the user did not override them.
* JDK classes, hadoop classes and resources, and some select third-party
* classes are considered system classes, and are not loaded by the
* application classloader.
*/
public static final String DEFAULT_SYSTEM_CLASSES =
"java.," +
"javax.," +
"org.w3c.dom.," +
"org.xml.sax.," +
"org.apache.commons.logging.," +
"org.apache.log4j.," +
"org.apache.hadoop.," +
"core-default.xml," +
"hdfs-default.xml," +
"mapred-default.xml," +
"yarn-default.xml";
private static final Log LOG =
LogFactory.getLog(ApplicationClassLoader.class.getName());
private static final FilenameFilter JAR_FILENAME_FILTER =
new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".jar") || name.endsWith(".JAR");
}
};
private final ClassLoader parent;
private final List<String> systemClasses;
public ApplicationClassLoader(URL[] urls, ClassLoader parent,
List<String> systemClasses) {
super(urls, parent);
if (LOG.isDebugEnabled()) {
LOG.debug("urls: " + Arrays.toString(urls));
LOG.debug("system classes: " + systemClasses);
}
this.parent = parent;
if (parent == null) {
throw new IllegalArgumentException("No parent classloader!");
}
// if the caller-specified system classes are null or empty, use the default
this.systemClasses = (systemClasses == null || systemClasses.isEmpty()) ?
Arrays.asList(StringUtils.getTrimmedStrings(DEFAULT_SYSTEM_CLASSES)) :
systemClasses;
LOG.info("system classes: " + this.systemClasses);
}
public ApplicationClassLoader(String classpath, ClassLoader parent,
List<String> systemClasses) throws MalformedURLException {
this(constructUrlsFromClasspath(classpath), parent, systemClasses);
}
static URL[] constructUrlsFromClasspath(String classpath)
throws MalformedURLException {
List<URL> urls = new ArrayList<URL>();
for (String element : classpath.split(File.pathSeparator)) {
if (element.endsWith("/*")) {
String dir = element.substring(0, element.length() - 1);
File[] files = new File(dir).listFiles(JAR_FILENAME_FILTER);
if (files != null) {
for (File file : files) {
urls.add(file.toURI().toURL());
}
}
} else {
File file = new File(element);
if (file.exists()) {
urls.add(new File(element).toURI().toURL());
}
}
}
return urls.toArray(new URL[urls.size()]);
}
@Override
public URL getResource(String name) {
URL url = null;
if (!isSystemClass(name, systemClasses)) {
url= findResource(name);
if (url == null && name.startsWith("/")) {
if (LOG.isDebugEnabled()) {
LOG.debug("Remove leading / off " + name);
}
url= findResource(name.substring(1));
}
}
if (url == null) {
url= parent.getResource(name);
}
if (url != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("getResource("+name+")=" + url);
}
}
return url;
}
@Override
public Class<?> loadClass(String name) throws ClassNotFoundException {
return this.loadClass(name, false);
}
@Override
protected synchronized Class<?> loadClass(String name, boolean resolve)
throws ClassNotFoundException {
if (LOG.isDebugEnabled()) {
LOG.debug("Loading class: " + name);
}
Class<?> c = findLoadedClass(name);
ClassNotFoundException ex = null;
if (c == null && !isSystemClass(name, systemClasses)) {
// Try to load class from this classloader's URLs. Note that this is like
// the servlet spec, not the usual Java 2 behaviour where we ask the
// parent to attempt to load first.
try {
c = findClass(name);
if (LOG.isDebugEnabled() && c != null) {
LOG.debug("Loaded class: " + name + " ");
}
} catch (ClassNotFoundException e) {
if (LOG.isDebugEnabled()) {
LOG.debug(e);
}
ex = e;
}
}
if (c == null) { // try parent
c = parent.loadClass(name);
if (LOG.isDebugEnabled() && c != null) {
LOG.debug("Loaded class from parent: " + name + " ");
}
}
if (c == null) {
throw ex != null ? ex : new ClassNotFoundException(name);
}
if (resolve) {
resolveClass(c);
}
return c;
}
public static boolean isSystemClass(String name, List<String> systemClasses) {
if (systemClasses != null) {
String canonicalName = name.replace('/', '.');
while (canonicalName.startsWith(".")) {
canonicalName=canonicalName.substring(1);
}
for (String c : systemClasses) {
boolean result = true;
if (c.startsWith("-")) {
c = c.substring(1);
result = false;
}
if (c.endsWith(".") && canonicalName.startsWith(c)) {
return result;
} else if (canonicalName.equals(c)) {
return result;
}
}
}
return false;
}
}

115
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
View File

@ -18,23 +18,25 @@
package org.apache.hadoop.util;
import java.lang.reflect.Array;
import java.lang.reflect.Method;
import java.lang.reflect.InvocationTargetException;
import java.net.URL;
import java.net.URLClassLoader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.File;
import java.util.regex.Pattern;
import java.util.Arrays;
import java.lang.reflect.Array;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.jar.JarFile;
import java.util.List;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.jar.Manifest;
import java.util.regex.Pattern;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@ -55,6 +57,21 @@ public class RunJar {
*/
public static final int SHUTDOWN_HOOK_PRIORITY = 10;
/**
* Environment key for using the client classloader.
*/
public static final String HADOOP_USE_CLIENT_CLASSLOADER =
"HADOOP_USE_CLIENT_CLASSLOADER";
/**
* Environment key for the (user-provided) hadoop classpath.
*/
public static final String HADOOP_CLASSPATH = "HADOOP_CLASSPATH";
/**
* Environment key for the system classes.
*/
public static final String HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES =
"HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES";
/**
* Unpack a jar file into a directory.
*
@ -116,6 +133,10 @@ public class RunJar {
/** Run a Hadoop job jar. If the main class is not in the jar's manifest,
* then it must be provided on the command line. */
public static void main(String[] args) throws Throwable {
new RunJar().run(args);
}
public void run(String[] args) throws Throwable {
String usage = "RunJar jarFile [mainClass] args...";
if (args.length < 1) {
@ -187,19 +208,7 @@ public class RunJar {
unJar(file, workDir);
ArrayList<URL> classPath = new ArrayList<URL>();
classPath.add(new File(workDir+"/").toURI().toURL());
classPath.add(file.toURI().toURL());
classPath.add(new File(workDir, "classes/").toURI().toURL());
File[] libs = new File(workDir, "lib").listFiles();
if (libs != null) {
for (int i = 0; i < libs.length; i++) {
classPath.add(libs[i].toURI().toURL());
}
}
ClassLoader loader =
new URLClassLoader(classPath.toArray(new URL[0]));
ClassLoader loader = createClassLoader(file, workDir);
Thread.currentThread().setContextClassLoader(loader);
Class<?> mainClass = Class.forName(mainClassName, true, loader);
@ -214,5 +223,65 @@ public class RunJar {
throw e.getTargetException();
}
}
/**
* Creates a classloader based on the environment that was specified by the
* user. If HADOOP_USE_CLIENT_CLASSLOADER is specified, it creates an
* application classloader that provides the isolation of the user class space
* from the hadoop classes and their dependencies. It forms a class space for
* the user jar as well as the HADOOP_CLASSPATH. Otherwise, it creates a
* classloader that simply adds the user jar to the classpath.
*/
private ClassLoader createClassLoader(File file, final File workDir)
throws MalformedURLException {
ClassLoader loader;
// see if the client classloader is enabled
if (useClientClassLoader()) {
StringBuilder sb = new StringBuilder();
sb.append(workDir+"/").
append(File.pathSeparator).append(file).
append(File.pathSeparator).append(workDir+"/classes/").
append(File.pathSeparator).append(workDir+"/lib/*");
// HADOOP_CLASSPATH is added to the client classpath
String hadoopClasspath = getHadoopClasspath();
if (hadoopClasspath != null && !hadoopClasspath.isEmpty()) {
sb.append(File.pathSeparator).append(hadoopClasspath);
}
String clientClasspath = sb.toString();
// get the system classes
String systemClasses = getSystemClasses();
List<String> systemClassesList = systemClasses == null ?
null :
Arrays.asList(StringUtils.getTrimmedStrings(systemClasses));
// create an application classloader that isolates the user classes
loader = new ApplicationClassLoader(clientClasspath,
getClass().getClassLoader(), systemClassesList);
} else {
List<URL> classPath = new ArrayList<URL>();
classPath.add(new File(workDir+"/").toURI().toURL());
classPath.add(file.toURI().toURL());
classPath.add(new File(workDir, "classes/").toURI().toURL());
File[] libs = new File(workDir, "lib").listFiles();
if (libs != null) {
for (int i = 0; i < libs.length; i++) {
classPath.add(libs[i].toURI().toURL());
}
}
// create a normal parent-delegating classloader
loader = new URLClassLoader(classPath.toArray(new URL[0]));
}
return loader;
}
boolean useClientClassLoader() {
return Boolean.parseBoolean(System.getenv(HADOOP_USE_CLIENT_CLASSLOADER));
}
String getHadoopClasspath() {
return System.getenv(HADOOP_CLASSPATH);
}
String getSystemClasses() {
return System.getenv(HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES);
}
}

33
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/ClassLoaderCheck.java Normal file
View File

@ -0,0 +1,33 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.util;
public class ClassLoaderCheck {
/**
* Verifies the class is loaded by the right classloader.
*/
public static void checkClassLoader(Class cls,
boolean shouldBeLoadedByAppClassLoader) {
boolean loadedByAppClassLoader =
cls.getClassLoader() instanceof ApplicationClassLoader;
if ((shouldBeLoadedByAppClassLoader && !loadedByAppClassLoader) ||
(!shouldBeLoadedByAppClassLoader && loadedByAppClassLoader)) {
throw new RuntimeException("incorrect classloader used");
}
}
}

34
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/ClassLoaderCheckMain.java Normal file
View File

@ -0,0 +1,34 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.util;
/**
* Test class used by {@link TestRunJar} to verify that it is loaded by the
* {@link ApplicationClassLoader}.
*/
public class ClassLoaderCheckMain {
public static void main(String[] args) {
// ClassLoaderCheckMain should be loaded by the application classloader
ClassLoaderCheck.checkClassLoader(ClassLoaderCheckMain.class, true);
// ClassLoaderCheckSecond should NOT be loaded by the application
// classloader
ClassLoaderCheck.checkClassLoader(ClassLoaderCheckSecond.class, false);
// ClassLoaderCheckThird should be loaded by the application classloader
ClassLoaderCheck.checkClassLoader(ClassLoaderCheckThird.class, true);
}
}

24
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/ClassLoaderCheckSecond.java Normal file
View File

@ -0,0 +1,24 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.util;
/**
* A class {@link ClassLoaderCheckMain} depends on that should be loaded by the
* system classloader.
*/
public class ClassLoaderCheckSecond {}

24
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/ClassLoaderCheckThird.java Normal file
View File

@ -0,0 +1,24 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.util;
/**
* A class {@link ClassLoaderCheckMain} depends on that should be loaded by the
* application classloader.
*/
public class ClassLoaderCheckThird {}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApplicationClassLoader.java → hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
View File

@ -16,18 +16,15 @@
* limitations under the License.
*/
package org.apache.hadoop.yarn.util;
package org.apache.hadoop.util;
import static org.apache.hadoop.util.ApplicationClassLoader.constructUrlsFromClasspath;
import static org.apache.hadoop.util.ApplicationClassLoader.isSystemClass;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.apache.hadoop.yarn.util.ApplicationClassLoader.constructUrlsFromClasspath;
import static org.apache.hadoop.yarn.util.ApplicationClassLoader.isSystemClass;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import java.io.File;
import java.io.FileOutputStream;
@ -43,6 +40,9 @@ import org.apache.hadoop.fs.FileUtil;
import org.junit.Before;
import org.junit.Test;
import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
public class TestApplicationClassLoader {
private static File testDir = new File(System.getProperty("test.build.data",

66
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestRunJar.java
View File

@ -17,23 +17,30 @@
*/
package org.apache.hadoop.util;
import junit.framework.TestCase;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.jar.JarOutputStream;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import junit.framework.TestCase;
import org.apache.hadoop.fs.FileUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.apache.hadoop.fs.FileUtil;
public class TestRunJar extends TestCase {
private File TEST_ROOT_DIR;
private static final String TEST_JAR_NAME="test-runjar.jar";
private static final String TEST_JAR_2_NAME = "test-runjar2.jar";
@Override
@Before
@ -107,4 +114,59 @@ public class TestRunJar extends TestCase {
new File(unjarDir, "foobaz.txt").exists());
}
/**
* Tests the client classloader to verify the main class and its dependent
* class are loaded correctly by the application classloader, and others are
* loaded by the system classloader.
*/
@Test
public void testClientClassLoader() throws Throwable {
RunJar runJar = spy(new RunJar());
// enable the client classloader
when(runJar.useClientClassLoader()).thenReturn(true);
// set the system classes and blacklist the test main class and the test
// third class so they can be loaded by the application classloader
String mainCls = ClassLoaderCheckMain.class.getName();
String thirdCls = ClassLoaderCheckThird.class.getName();
String systemClasses = "-" + mainCls + "," +
"-" + thirdCls + "," +
ApplicationClassLoader.DEFAULT_SYSTEM_CLASSES;
when(runJar.getSystemClasses()).thenReturn(systemClasses);
// create the test jar
File testJar = makeClassLoaderTestJar(mainCls, thirdCls);
// form the args
String[] args = new String[3];
args[0] = testJar.getAbsolutePath();
args[1] = mainCls;
// run RunJar
runJar.run(args);
// it should not throw an exception
}
private File makeClassLoaderTestJar(String... clsNames) throws IOException {
File jarFile = new File(TEST_ROOT_DIR, TEST_JAR_2_NAME);
JarOutputStream jstream =
new JarOutputStream(new FileOutputStream(jarFile));
for (String clsName: clsNames) {
String name = clsName.replace('.', '/') + ".class";
InputStream entryInputStream = this.getClass().getResourceAsStream(
"/" + name);
ZipEntry entry = new ZipEntry(name);
jstream.putNextEntry(entry);
BufferedInputStream bufInputStream = new BufferedInputStream(
entryInputStream, 2048);
int count;
byte[] data = new byte[2048];
while ((count = bufInputStream.read(data, 0, 2048)) != -1) {
jstream.write(data, 0, count);
}
jstream.closeEntry();
}
jstream.close();
return jarFile;
}
}

3
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
View File

@ -34,6 +34,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@ -56,6 +57,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.util.ApplicationClassLoader;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
@ -67,7 +69,6 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.util.ApplicationClassLoader;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.log4j.RollingFileAppender;

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
View File

@ -51,6 +51,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.util.ApplicationClassLoader;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -58,7 +59,6 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.util.ApplicationClassLoader;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@ -514,7 +514,8 @@ public class TestMRApps {
@Test
public void testSystemClasses() {
final List<String> systemClasses =
Arrays.asList(MRApps.getSystemClasses(new Configuration()));
Arrays.asList(StringUtils.getTrimmedStrings(
ApplicationClassLoader.DEFAULT_SYSTEM_CLASSES));
for (String defaultXml : DEFAULT_XMLS) {
assertTrue(defaultXml + " must be system resource",
ApplicationClassLoader.isSystemClass(defaultXml, systemClasses));

14
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
View File

@ -1227,13 +1227,13 @@
<property>
<name>mapreduce.job.classloader.system.classes</name>
<value>java.,javax.,org.w3c.dom.,org.xml.sax.,org.apache.commons.logging.,
org.apache.log4j.,org.apache.hadoop.,core-default.xml,
hdfs-default.xml,mapred-default.xml,yarn-default.xml</value>
<description>A comma-separated list of classes that should be loaded from the
system classpath, not the user-supplied JARs, when mapreduce.job.classloader
is enabled. Names ending in '.' (period) are treated as package names,
and names starting with a '-' are treated as negative matches.
<value></value>
<description>Used to override the default definition of the system classes for
the job classloader. The system classes are a comma-separated list of
classes that should be loaded from the system classpath, not the
user-supplied JARs, when mapreduce.job.classloader is enabled. Names ending
in '.' (period) are treated as package names, and names starting with a '-'
are treated as negative matches.
</description>
</property>

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java
View File

@ -84,13 +84,13 @@ import org.apache.hadoop.mapreduce.v2.app.speculate.Speculator;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.ApplicationClassLoader;
import org.apache.hadoop.util.JarFinder;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.util.ApplicationClassLoader;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.log4j.Level;
import org.junit.AfterClass;
@ -242,8 +242,7 @@ public class TestMRJobs {
// to test AM loading user classes such as output format class, we want
// to blacklist them from the system classes (they need to be prepended
// as the first match wins)
String systemClasses =
sleepConf.get(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES);
String systemClasses = ApplicationClassLoader.DEFAULT_SYSTEM_CLASSES;
// exclude the custom classes from system classes
systemClasses = "-" + CustomOutputFormat.class.getName() + ",-" +
CustomSpeculator.class.getName() + "," +

7
hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
View File

@ -344,4 +344,11 @@
<Class name="org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider"/>
<Bug pattern="DC_DOUBLECHECK" />
</Match>
<!-- ApplicationClassLoader is deprecated and moved to hadoop-common; ignore
warning on the identical name as it should be removed later -->
<Match>
<Class name="org.apache.hadoop.yarn.util.ApplicationClassLoader"/>
<Bug pattern="NM_SAME_SIMPLE_NAME_AS_SUPERCLASS"/>
</Match>
</FindBugsFilter>

170
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
View File

@ -18,180 +18,30 @@
package org.apache.hadoop.yarn.util;
import java.io.File;
import java.io.FilenameFilter;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Splitter;
/**
* A {@link URLClassLoader} for YARN application isolation. Classes from
* the application JARs are loaded in preference to the parent loader.
* This type has been deprecated in favor of
* {@link org.apache.hadoop.util.ApplicationClassLoader}. All new uses of
* ApplicationClassLoader should use that type instead.
*/
@Public
@Unstable
public class ApplicationClassLoader extends URLClassLoader {
private static final Log LOG =
LogFactory.getLog(ApplicationClassLoader.class.getName());
private static final FilenameFilter JAR_FILENAME_FILTER =
new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".jar") || name.endsWith(".JAR");
}
};
private ClassLoader parent;
private List<String> systemClasses;
@Deprecated
public class ApplicationClassLoader extends
org.apache.hadoop.util.ApplicationClassLoader {
public ApplicationClassLoader(URL[] urls, ClassLoader parent,
List<String> systemClasses) {
super(urls, parent);
this.parent = parent;
if (parent == null) {
throw new IllegalArgumentException("No parent classloader!");
}
this.systemClasses = systemClasses;
super(urls, parent, systemClasses);
}
public ApplicationClassLoader(String classpath, ClassLoader parent,
List<String> systemClasses) throws MalformedURLException {
this(constructUrlsFromClasspath(classpath), parent, systemClasses);
super(classpath, parent, systemClasses);
}
@VisibleForTesting
static URL[] constructUrlsFromClasspath(String classpath)
throws MalformedURLException {
List<URL> urls = new ArrayList<URL>();
for (String element : Splitter.on(File.pathSeparator).split(classpath)) {
if (element.endsWith("/*")) {
String dir = element.substring(0, element.length() - 1);
File[] files = new File(dir).listFiles(JAR_FILENAME_FILTER);
if (files != null) {
for (File file : files) {
urls.add(file.toURI().toURL());
}
}
} else {
File file = new File(element);
if (file.exists()) {
urls.add(new File(element).toURI().toURL());
}
}
}
return urls.toArray(new URL[urls.size()]);
}
@Override
public URL getResource(String name) {
URL url = null;
if (!isSystemClass(name, systemClasses)) {
url= findResource(name);
if (url == null && name.startsWith("/")) {
if (LOG.isDebugEnabled()) {
LOG.debug("Remove leading / off " + name);
}
url= findResource(name.substring(1));
}
}
if (url == null) {
url= parent.getResource(name);
}
if (url != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("getResource("+name+")=" + url);
}
}
return url;
}
@Override
public Class<?> loadClass(String name) throws ClassNotFoundException {
return this.loadClass(name, false);
}
@Override
protected synchronized Class<?> loadClass(String name, boolean resolve)
throws ClassNotFoundException {
if (LOG.isDebugEnabled()) {
LOG.debug("Loading class: " + name);
}
Class<?> c = findLoadedClass(name);
ClassNotFoundException ex = null;
if (c == null && !isSystemClass(name, systemClasses)) {
// Try to load class from this classloader's URLs. Note that this is like
// the servlet spec, not the usual Java 2 behaviour where we ask the
// parent to attempt to load first.
try {
c = findClass(name);
if (LOG.isDebugEnabled() && c != null) {
LOG.debug("Loaded class: " + name + " ");
}
} catch (ClassNotFoundException e) {
if (LOG.isDebugEnabled()) {
LOG.debug(e);
}
ex = e;
}
}
if (c == null) { // try parent
c = parent.loadClass(name);
if (LOG.isDebugEnabled() && c != null) {
LOG.debug("Loaded class from parent: " + name + " ");
}
}
if (c == null) {
throw ex != null ? ex : new ClassNotFoundException(name);
}
if (resolve) {
resolveClass(c);
}
return c;
}
@VisibleForTesting
public static boolean isSystemClass(String name, List<String> systemClasses) {
if (systemClasses != null) {
String canonicalName = name.replace('/', '.');
while (canonicalName.startsWith(".")) {
canonicalName=canonicalName.substring(1);
}
for (String c : systemClasses) {
boolean result = true;
if (c.startsWith("-")) {
c = c.substring(1);
result = false;
}
if (c.endsWith(".") && canonicalName.startsWith(c)) {
return result;
} else if (canonicalName.equals(c)) {
return result;
}
}
}
return false;
}
}
}