From c77f41224dffd925f04666df8ae104229e2e568b Mon Sep 17 00:00:00 2001 From: Thomas White Date: Fri, 4 Jan 2013 11:02:51 +0000 Subject: [PATCH] Merge -r 1428780:1428781 from trunk to branch-2. Fixes: YARN-286. Add a YARN ApplicationClassLoader. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1428782 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 + .../hadoop/yarn/api/ApplicationConstants.java | 5 + .../yarn/util/ApplicationClassLoader.java | 197 ++++++++++++++++++ .../yarn/util/TestApplicationClassLoader.java | 136 ++++++++++++ 4 files changed, 340 insertions(+) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApplicationClassLoader.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index d527c63f8e0..d4d9b022a40 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -13,6 +13,8 @@ Release 2.0.3-alpha - Unreleased YARN-103. Add a yarn AM-RM client module. (Bikas Saha via sseth) + YARN-286. Add a YARN ApplicationClassLoader. (tomwhite) + IMPROVEMENTS YARN-223. Update process tree instead of getting new process trees. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index 7431067fa80..c86b3f9b86f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -142,6 +142,11 @@ public interface ApplicationConstants { */ CLASSPATH("CLASSPATH"), + /** + * $APP_CLASSPATH + */ + APP_CLASSPATH("APP_CLASSPATH"), + /** * $LD_LIBRARY_PATH */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java new file mode 100644 index 00000000000..8ef9e4a8063 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Splitter; + +import java.io.File; +import java.io.FilenameFilter; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * A {@link URLClassLoader} for YARN application isolation. Classes from + * the application JARs are loaded in preference to the parent loader. + */ +@Public +@Unstable +public class ApplicationClassLoader extends URLClassLoader { + + private static final Log LOG = + LogFactory.getLog(ApplicationClassLoader.class.getName()); + + private static final FilenameFilter JAR_FILENAME_FILTER = + new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".jar") || name.endsWith(".JAR"); + } + }; + + private ClassLoader parent; + private List systemClasses; + + public ApplicationClassLoader(URL[] urls, ClassLoader parent, + List systemClasses) { + super(urls, parent); + this.parent = parent; + if (parent == null) { + throw new IllegalArgumentException("No parent classloader!"); + } + this.systemClasses = systemClasses; + } + + public ApplicationClassLoader(String classpath, ClassLoader parent, + List systemClasses) throws MalformedURLException { + this(constructUrlsFromClasspath(classpath), parent, systemClasses); + } + + @VisibleForTesting + static URL[] constructUrlsFromClasspath(String classpath) + throws MalformedURLException { + List urls = new ArrayList(); + for (String element : Splitter.on(File.pathSeparator).split(classpath)) { + if (element.endsWith("/*")) { + String dir = element.substring(0, element.length() - 1); + File[] files = new File(dir).listFiles(JAR_FILENAME_FILTER); + if (files != null) { + for (File file : files) { + urls.add(file.toURI().toURL()); + } + } + } else { + File file = new File(element); + if (file.exists()) { + urls.add(new File(element).toURI().toURL()); + } + } + } + return urls.toArray(new URL[urls.size()]); + } + + @Override + public URL getResource(String name) { + URL url = null; + + if (!isSystemClass(name, systemClasses)) { + url= findResource(name); + if (url == null && name.startsWith("/")) { + if (LOG.isDebugEnabled()) { + LOG.debug("Remove leading / off " + name); + } + url= findResource(name.substring(1)); + } + } + + if (url == null) { + url= parent.getResource(name); + } + + if (url != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("getResource("+name+")=" + url); + } + } + + return url; + } + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + return this.loadClass(name, false); + } + + @Override + protected synchronized Class loadClass(String name, boolean resolve) + throws ClassNotFoundException { + + if (LOG.isDebugEnabled()) { + LOG.debug("Loading class: " + name); + } + + Class c = findLoadedClass(name); + ClassNotFoundException ex = null; + + if (c == null && !isSystemClass(name, systemClasses)) { + // Try to load class from this classloader's URLs. Note that this is like + // the servlet spec, not the usual Java 2 behaviour where we ask the + // parent to attempt to load first. + try { + c = findClass(name); + if (LOG.isDebugEnabled() && c != null) { + LOG.debug("Loaded class: " + name + " "); + } + } catch (ClassNotFoundException e) { + if (LOG.isDebugEnabled()) { + LOG.debug(e); + } + ex = e; + } + } + + if (c == null) { // try parent + c = parent.loadClass(name); + if (LOG.isDebugEnabled() && c != null) { + LOG.debug("Loaded class from parent: " + name + " "); + } + } + + if (c == null) { + throw ex != null ? ex : new ClassNotFoundException(name); + } + + if (resolve) { + resolveClass(c); + } + + return c; + } + + @VisibleForTesting + static boolean isSystemClass(String name, List systemClasses) { + if (systemClasses != null) { + String canonicalName = name.replace('/', '.'); + while (canonicalName.startsWith(".")) { + canonicalName=canonicalName.substring(1); + } + for (String c : systemClasses) { + boolean result = true; + if (c.startsWith("-")) { + c = c.substring(1); + result = false; + } + if (c.endsWith(".") && canonicalName.startsWith(c)) { + return result; + } else if (canonicalName.equals(c)) { + return result; + } + } + } + return false; + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApplicationClassLoader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApplicationClassLoader.java new file mode 100644 index 00000000000..abc7873a9ea --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApplicationClassLoader.java @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertFalse; +import static junit.framework.Assert.assertNotNull; +import static junit.framework.Assert.assertNull; +import static junit.framework.Assert.assertTrue; +import static org.apache.hadoop.yarn.util.ApplicationClassLoader.constructUrlsFromClasspath; +import static org.apache.hadoop.yarn.util.ApplicationClassLoader.isSystemClass; + +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.List; +import java.util.jar.JarOutputStream; +import java.util.zip.ZipEntry; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.fs.FileUtil; +import org.junit.Before; +import org.junit.Test; + +public class TestApplicationClassLoader { + + private static File testDir = new File(System.getProperty("test.build.data", + System.getProperty("java.io.tmpdir")), "appclassloader"); + + @Before + public void setUp() { + FileUtil.fullyDelete(testDir); + testDir.mkdirs(); + } + + @Test + public void testConstructUrlsFromClasspath() throws Exception { + File file = new File(testDir, "file"); + assertTrue("Create file", file.createNewFile()); + + File dir = new File(testDir, "dir"); + assertTrue("Make dir", dir.mkdir()); + + File jarsDir = new File(testDir, "jarsdir"); + assertTrue("Make jarsDir", jarsDir.mkdir()); + File nonJarFile = new File(jarsDir, "nonjar"); + assertTrue("Create non-jar file", nonJarFile.createNewFile()); + File jarFile = new File(jarsDir, "a.jar"); + assertTrue("Create jar file", jarFile.createNewFile()); + + File nofile = new File(testDir, "nofile"); + // don't create nofile + + StringBuilder cp = new StringBuilder(); + cp.append(file.getAbsolutePath()).append(File.pathSeparator) + .append(dir.getAbsolutePath()).append(File.pathSeparator) + .append(jarsDir.getAbsolutePath() + "/*").append(File.pathSeparator) + .append(nofile.getAbsolutePath()).append(File.pathSeparator) + .append(nofile.getAbsolutePath() + "/*").append(File.pathSeparator); + + URL[] urls = constructUrlsFromClasspath(cp.toString()); + + assertEquals(3, urls.length); + assertEquals(file.toURI().toURL(), urls[0]); + assertEquals(dir.toURI().toURL(), urls[1]); + assertEquals(jarFile.toURI().toURL(), urls[2]); + // nofile should be ignored + } + + @Test + public void testIsSystemClass() { + assertFalse(isSystemClass("org.example.Foo", null)); + assertTrue(isSystemClass("org.example.Foo", classes("org.example.Foo"))); + assertTrue(isSystemClass("/org.example.Foo", classes("org.example.Foo"))); + assertTrue(isSystemClass("org.example.Foo", classes("org.example."))); + assertTrue(isSystemClass("net.example.Foo", + classes("org.example.,net.example."))); + assertFalse(isSystemClass("org.example.Foo", + classes("-org.example.Foo,org.example."))); + assertTrue(isSystemClass("org.example.Bar", + classes("-org.example.Foo.,org.example."))); + } + + private List classes(String classes) { + return Lists.newArrayList(Splitter.on(',').split(classes)); + } + + @Test + public void testGetResource() throws IOException { + URL testJar = makeTestJar().toURI().toURL(); + + ClassLoader currentClassLoader = getClass().getClassLoader(); + ClassLoader appClassloader = new ApplicationClassLoader( + new URL[] { testJar }, currentClassLoader, null); + + assertNull("Resource should be null for current classloader", + currentClassLoader.getResourceAsStream("resource.txt")); + + InputStream in = appClassloader.getResourceAsStream("resource.txt"); + assertNotNull("Resource should not be null for app classloader", in); + assertEquals("hello", IOUtils.toString(in)); + } + + private File makeTestJar() throws IOException { + File jarFile = new File(testDir, "test.jar"); + JarOutputStream out = new JarOutputStream(new FileOutputStream(jarFile)); + ZipEntry entry = new ZipEntry("resource.txt"); + out.putNextEntry(entry); + out.write("hello".getBytes()); + out.closeEntry(); + out.close(); + return jarFile; + } + +}