Merge -r 1428780:1428781 from trunk to branch-2. Fixes: YARN-286. Add a YARN ApplicationClassLoader.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1428782 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Thomas White
parent
3e68d15c1e
commit
c77f41224d
|
|
@ -13,6 +13,8 @@ Release 2.0.3-alpha - Unreleased
|
||||||
|
|
||||||
YARN-103. Add a yarn AM-RM client module. (Bikas Saha via sseth)
|
YARN-103. Add a yarn AM-RM client module. (Bikas Saha via sseth)
|
||||||
|
|
||||||
|
YARN-286. Add a YARN ApplicationClassLoader. (tomwhite)
|
||||||
|
|
||||||
IMPROVEMENTS
|
IMPROVEMENTS
|
||||||
|
|
||||||
YARN-223. Update process tree instead of getting new process trees.
|
YARN-223. Update process tree instead of getting new process trees.
|
||||||
|
|
|
||||||
|
|
@ -142,6 +142,11 @@ public interface ApplicationConstants {
|
||||||
*/
|
*/
|
||||||
CLASSPATH("CLASSPATH"),
|
CLASSPATH("CLASSPATH"),
|
||||||
|
|
||||||
|
/**
|
||||||
|
* $APP_CLASSPATH
|
||||||
|
*/
|
||||||
|
APP_CLASSPATH("APP_CLASSPATH"),
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* $LD_LIBRARY_PATH
|
* $LD_LIBRARY_PATH
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,197 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.util;
|
||||||
|
|
||||||
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
import com.google.common.base.Splitter;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FilenameFilter;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.net.URLClassLoader;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.commons.logging.Log;
|
||||||
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience.Public;
|
||||||
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A {@link URLClassLoader} for YARN application isolation. Classes from
|
||||||
|
* the application JARs are loaded in preference to the parent loader.
|
||||||
|
*/
|
||||||
|
@Public
|
||||||
|
@Unstable
|
||||||
|
public class ApplicationClassLoader extends URLClassLoader {
|
||||||
|
|
||||||
|
private static final Log LOG =
|
||||||
|
LogFactory.getLog(ApplicationClassLoader.class.getName());
|
||||||
|
|
||||||
|
private static final FilenameFilter JAR_FILENAME_FILTER =
|
||||||
|
new FilenameFilter() {
|
||||||
|
@Override
|
||||||
|
public boolean accept(File dir, String name) {
|
||||||
|
return name.endsWith(".jar") || name.endsWith(".JAR");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
private ClassLoader parent;
|
||||||
|
private List<String> systemClasses;
|
||||||
|
|
||||||
|
public ApplicationClassLoader(URL[] urls, ClassLoader parent,
|
||||||
|
List<String> systemClasses) {
|
||||||
|
super(urls, parent);
|
||||||
|
this.parent = parent;
|
||||||
|
if (parent == null) {
|
||||||
|
throw new IllegalArgumentException("No parent classloader!");
|
||||||
|
}
|
||||||
|
this.systemClasses = systemClasses;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ApplicationClassLoader(String classpath, ClassLoader parent,
|
||||||
|
List<String> systemClasses) throws MalformedURLException {
|
||||||
|
this(constructUrlsFromClasspath(classpath), parent, systemClasses);
|
||||||
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
static URL[] constructUrlsFromClasspath(String classpath)
|
||||||
|
throws MalformedURLException {
|
||||||
|
List<URL> urls = new ArrayList<URL>();
|
||||||
|
for (String element : Splitter.on(File.pathSeparator).split(classpath)) {
|
||||||
|
if (element.endsWith("/*")) {
|
||||||
|
String dir = element.substring(0, element.length() - 1);
|
||||||
|
File[] files = new File(dir).listFiles(JAR_FILENAME_FILTER);
|
||||||
|
if (files != null) {
|
||||||
|
for (File file : files) {
|
||||||
|
urls.add(file.toURI().toURL());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
File file = new File(element);
|
||||||
|
if (file.exists()) {
|
||||||
|
urls.add(new File(element).toURI().toURL());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return urls.toArray(new URL[urls.size()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public URL getResource(String name) {
|
||||||
|
URL url = null;
|
||||||
|
|
||||||
|
if (!isSystemClass(name, systemClasses)) {
|
||||||
|
url= findResource(name);
|
||||||
|
if (url == null && name.startsWith("/")) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Remove leading / off " + name);
|
||||||
|
}
|
||||||
|
url= findResource(name.substring(1));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (url == null) {
|
||||||
|
url= parent.getResource(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (url != null) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("getResource("+name+")=" + url);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Class<?> loadClass(String name) throws ClassNotFoundException {
|
||||||
|
return this.loadClass(name, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected synchronized Class<?> loadClass(String name, boolean resolve)
|
||||||
|
throws ClassNotFoundException {
|
||||||
|
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Loading class: " + name);
|
||||||
|
}
|
||||||
|
|
||||||
|
Class<?> c = findLoadedClass(name);
|
||||||
|
ClassNotFoundException ex = null;
|
||||||
|
|
||||||
|
if (c == null && !isSystemClass(name, systemClasses)) {
|
||||||
|
// Try to load class from this classloader's URLs. Note that this is like
|
||||||
|
// the servlet spec, not the usual Java 2 behaviour where we ask the
|
||||||
|
// parent to attempt to load first.
|
||||||
|
try {
|
||||||
|
c = findClass(name);
|
||||||
|
if (LOG.isDebugEnabled() && c != null) {
|
||||||
|
LOG.debug("Loaded class: " + name + " ");
|
||||||
|
}
|
||||||
|
} catch (ClassNotFoundException e) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug(e);
|
||||||
|
}
|
||||||
|
ex = e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (c == null) { // try parent
|
||||||
|
c = parent.loadClass(name);
|
||||||
|
if (LOG.isDebugEnabled() && c != null) {
|
||||||
|
LOG.debug("Loaded class from parent: " + name + " ");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (c == null) {
|
||||||
|
throw ex != null ? ex : new ClassNotFoundException(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (resolve) {
|
||||||
|
resolveClass(c);
|
||||||
|
}
|
||||||
|
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
static boolean isSystemClass(String name, List<String> systemClasses) {
|
||||||
|
if (systemClasses != null) {
|
||||||
|
String canonicalName = name.replace('/', '.');
|
||||||
|
while (canonicalName.startsWith(".")) {
|
||||||
|
canonicalName=canonicalName.substring(1);
|
||||||
|
}
|
||||||
|
for (String c : systemClasses) {
|
||||||
|
boolean result = true;
|
||||||
|
if (c.startsWith("-")) {
|
||||||
|
c = c.substring(1);
|
||||||
|
result = false;
|
||||||
|
}
|
||||||
|
if (c.endsWith(".") && canonicalName.startsWith(c)) {
|
||||||
|
return result;
|
||||||
|
} else if (canonicalName.equals(c)) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,136 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.util;
|
||||||
|
|
||||||
|
import static junit.framework.Assert.assertEquals;
|
||||||
|
import static junit.framework.Assert.assertFalse;
|
||||||
|
import static junit.framework.Assert.assertNotNull;
|
||||||
|
import static junit.framework.Assert.assertNull;
|
||||||
|
import static junit.framework.Assert.assertTrue;
|
||||||
|
import static org.apache.hadoop.yarn.util.ApplicationClassLoader.constructUrlsFromClasspath;
|
||||||
|
import static org.apache.hadoop.yarn.util.ApplicationClassLoader.isSystemClass;
|
||||||
|
|
||||||
|
import com.google.common.base.Splitter;
|
||||||
|
import com.google.common.collect.Lists;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.jar.JarOutputStream;
|
||||||
|
import java.util.zip.ZipEntry;
|
||||||
|
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.apache.hadoop.fs.FileUtil;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class TestApplicationClassLoader {
|
||||||
|
|
||||||
|
private static File testDir = new File(System.getProperty("test.build.data",
|
||||||
|
System.getProperty("java.io.tmpdir")), "appclassloader");
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setUp() {
|
||||||
|
FileUtil.fullyDelete(testDir);
|
||||||
|
testDir.mkdirs();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testConstructUrlsFromClasspath() throws Exception {
|
||||||
|
File file = new File(testDir, "file");
|
||||||
|
assertTrue("Create file", file.createNewFile());
|
||||||
|
|
||||||
|
File dir = new File(testDir, "dir");
|
||||||
|
assertTrue("Make dir", dir.mkdir());
|
||||||
|
|
||||||
|
File jarsDir = new File(testDir, "jarsdir");
|
||||||
|
assertTrue("Make jarsDir", jarsDir.mkdir());
|
||||||
|
File nonJarFile = new File(jarsDir, "nonjar");
|
||||||
|
assertTrue("Create non-jar file", nonJarFile.createNewFile());
|
||||||
|
File jarFile = new File(jarsDir, "a.jar");
|
||||||
|
assertTrue("Create jar file", jarFile.createNewFile());
|
||||||
|
|
||||||
|
File nofile = new File(testDir, "nofile");
|
||||||
|
// don't create nofile
|
||||||
|
|
||||||
|
StringBuilder cp = new StringBuilder();
|
||||||
|
cp.append(file.getAbsolutePath()).append(File.pathSeparator)
|
||||||
|
.append(dir.getAbsolutePath()).append(File.pathSeparator)
|
||||||
|
.append(jarsDir.getAbsolutePath() + "/*").append(File.pathSeparator)
|
||||||
|
.append(nofile.getAbsolutePath()).append(File.pathSeparator)
|
||||||
|
.append(nofile.getAbsolutePath() + "/*").append(File.pathSeparator);
|
||||||
|
|
||||||
|
URL[] urls = constructUrlsFromClasspath(cp.toString());
|
||||||
|
|
||||||
|
assertEquals(3, urls.length);
|
||||||
|
assertEquals(file.toURI().toURL(), urls[0]);
|
||||||
|
assertEquals(dir.toURI().toURL(), urls[1]);
|
||||||
|
assertEquals(jarFile.toURI().toURL(), urls[2]);
|
||||||
|
// nofile should be ignored
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testIsSystemClass() {
|
||||||
|
assertFalse(isSystemClass("org.example.Foo", null));
|
||||||
|
assertTrue(isSystemClass("org.example.Foo", classes("org.example.Foo")));
|
||||||
|
assertTrue(isSystemClass("/org.example.Foo", classes("org.example.Foo")));
|
||||||
|
assertTrue(isSystemClass("org.example.Foo", classes("org.example.")));
|
||||||
|
assertTrue(isSystemClass("net.example.Foo",
|
||||||
|
classes("org.example.,net.example.")));
|
||||||
|
assertFalse(isSystemClass("org.example.Foo",
|
||||||
|
classes("-org.example.Foo,org.example.")));
|
||||||
|
assertTrue(isSystemClass("org.example.Bar",
|
||||||
|
classes("-org.example.Foo.,org.example.")));
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<String> classes(String classes) {
|
||||||
|
return Lists.newArrayList(Splitter.on(',').split(classes));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGetResource() throws IOException {
|
||||||
|
URL testJar = makeTestJar().toURI().toURL();
|
||||||
|
|
||||||
|
ClassLoader currentClassLoader = getClass().getClassLoader();
|
||||||
|
ClassLoader appClassloader = new ApplicationClassLoader(
|
||||||
|
new URL[] { testJar }, currentClassLoader, null);
|
||||||
|
|
||||||
|
assertNull("Resource should be null for current classloader",
|
||||||
|
currentClassLoader.getResourceAsStream("resource.txt"));
|
||||||
|
|
||||||
|
InputStream in = appClassloader.getResourceAsStream("resource.txt");
|
||||||
|
assertNotNull("Resource should not be null for app classloader", in);
|
||||||
|
assertEquals("hello", IOUtils.toString(in));
|
||||||
|
}
|
||||||
|
|
||||||
|
private File makeTestJar() throws IOException {
|
||||||
|
File jarFile = new File(testDir, "test.jar");
|
||||||
|
JarOutputStream out = new JarOutputStream(new FileOutputStream(jarFile));
|
||||||
|
ZipEntry entry = new ZipEntry("resource.txt");
|
||||||
|
out.putNextEntry(entry);
|
||||||
|
out.write("hello".getBytes());
|
||||||
|
out.closeEntry();
|
||||||
|
out.close();
|
||||||
|
return jarFile;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue