Merge -r 1428780:1428781 from trunk to branch-2. Fixes: YARN-286. Add a YARN ApplicationClassLoader.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1428782 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -13,6 +13,8 @@ Release 2.0.3-alpha - Unreleased
 
     YARN-103. Add a yarn AM-RM client module. (Bikas Saha via sseth)
 
+    YARN-286. Add a YARN ApplicationClassLoader. (tomwhite)
+
   IMPROVEMENTS
 
     YARN-223. Update process tree instead of getting new process trees.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java

@@ -142,6 +142,11 @@ public interface ApplicationConstants {
      */
     CLASSPATH("CLASSPATH"),
     
+    /**
+     * $APP_CLASSPATH
+     */
+    APP_CLASSPATH("APP_CLASSPATH"),
+    
     /**
      * $LD_LIBRARY_PATH
      */

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java

@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Splitter;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * A {@link URLClassLoader} for YARN application isolation. Classes from
+ * the application JARs are loaded in preference to the parent loader.
+ */
+@Public
+@Unstable
+public class ApplicationClassLoader extends URLClassLoader {
+
+  private static final Log LOG =
+    LogFactory.getLog(ApplicationClassLoader.class.getName());
+  
+  private static final FilenameFilter JAR_FILENAME_FILTER =
+    new FilenameFilter() {
+      @Override
+      public boolean accept(File dir, String name) {
+        return name.endsWith(".jar") || name.endsWith(".JAR");
+      }
+  };
+  
+  private ClassLoader parent;
+  private List<String> systemClasses;
+
+  public ApplicationClassLoader(URL[] urls, ClassLoader parent,
+      List<String> systemClasses) {
+    super(urls, parent);
+    this.parent = parent;
+    if (parent == null) {
+      throw new IllegalArgumentException("No parent classloader!");
+    }
+    this.systemClasses = systemClasses;
+  }
+  
+  public ApplicationClassLoader(String classpath, ClassLoader parent,
+      List<String> systemClasses) throws MalformedURLException {
+    this(constructUrlsFromClasspath(classpath), parent, systemClasses);
+  }
+  
+  @VisibleForTesting
+  static URL[] constructUrlsFromClasspath(String classpath)
+      throws MalformedURLException {
+    List<URL> urls = new ArrayList<URL>();
+    for (String element : Splitter.on(File.pathSeparator).split(classpath)) {
+      if (element.endsWith("/*")) {
+        String dir = element.substring(0, element.length() - 1);
+        File[] files = new File(dir).listFiles(JAR_FILENAME_FILTER);
+        if (files != null) {
+          for (File file : files) {
+            urls.add(file.toURI().toURL());
+          }
+        }
+      } else {
+        File file = new File(element);
+        if (file.exists()) {
+          urls.add(new File(element).toURI().toURL());
+        }
+      }
+    }
+    return urls.toArray(new URL[urls.size()]);
+  }
+
+  @Override
+  public URL getResource(String name) {
+    URL url = null;
+    
+    if (!isSystemClass(name, systemClasses)) {
+      url= findResource(name);
+      if (url == null && name.startsWith("/")) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Remove leading / off " + name);
+        }
+        url= findResource(name.substring(1));
+      }
+    }
+
+    if (url == null) {
+      url= parent.getResource(name);
+    }
+
+    if (url != null) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("getResource("+name+")=" + url);
+      }
+    }
+    
+    return url;
+  }
+
+  @Override
+  public Class<?> loadClass(String name) throws ClassNotFoundException {
+    return this.loadClass(name, false);
+  }
+
+  @Override
+  protected synchronized Class<?> loadClass(String name, boolean resolve)
+      throws ClassNotFoundException {
+    
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Loading class: " + name);
+    }
+
+    Class<?> c = findLoadedClass(name);
+    ClassNotFoundException ex = null;
+
+    if (c == null && !isSystemClass(name, systemClasses)) {
+      // Try to load class from this classloader's URLs. Note that this is like
+      // the servlet spec, not the usual Java 2 behaviour where we ask the
+      // parent to attempt to load first.
+      try {
+        c = findClass(name);
+        if (LOG.isDebugEnabled() && c != null) {
+          LOG.debug("Loaded class: " + name + " ");
+        }
+      } catch (ClassNotFoundException e) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug(e);
+        }
+        ex = e;
+      }
+    }
+
+    if (c == null) { // try parent
+      c = parent.loadClass(name);
+      if (LOG.isDebugEnabled() && c != null) {
+        LOG.debug("Loaded class from parent: " + name + " ");
+      }
+    }
+
+    if (c == null) {
+      throw ex != null ? ex : new ClassNotFoundException(name);
+    }
+
+    if (resolve) {
+      resolveClass(c);
+    }
+
+    return c;
+  }
+
+  @VisibleForTesting
+  static boolean isSystemClass(String name, List<String> systemClasses) {
+    if (systemClasses != null) {
+      String canonicalName = name.replace('/', '.');
+      while (canonicalName.startsWith(".")) {
+        canonicalName=canonicalName.substring(1);
+      }
+      for (String c : systemClasses) {
+        boolean result = true;
+        if (c.startsWith("-")) {
+          c = c.substring(1);
+          result = false;
+        }
+        if (c.endsWith(".") && canonicalName.startsWith(c)) {
+          return result;
+        } else if (canonicalName.equals(c)) {
+          return result;
+        }
+      }
+    }
+    return false;
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApplicationClassLoader.java

@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertFalse;
+import static junit.framework.Assert.assertNotNull;
+import static junit.framework.Assert.assertNull;
+import static junit.framework.Assert.assertTrue;
+import static org.apache.hadoop.yarn.util.ApplicationClassLoader.constructUrlsFromClasspath;
+import static org.apache.hadoop.yarn.util.ApplicationClassLoader.isSystemClass;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.List;
+import java.util.jar.JarOutputStream;
+import java.util.zip.ZipEntry;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestApplicationClassLoader {
+  
+  private static File testDir = new File(System.getProperty("test.build.data",
+          System.getProperty("java.io.tmpdir")), "appclassloader");
+  
+  @Before
+  public void setUp() {
+    FileUtil.fullyDelete(testDir);
+    testDir.mkdirs();
+  }
+
+  @Test
+  public void testConstructUrlsFromClasspath() throws Exception {
+    File file = new File(testDir, "file");
+    assertTrue("Create file", file.createNewFile());
+
+    File dir = new File(testDir, "dir");
+    assertTrue("Make dir", dir.mkdir());
+
+    File jarsDir = new File(testDir, "jarsdir");
+    assertTrue("Make jarsDir", jarsDir.mkdir());
+    File nonJarFile = new File(jarsDir, "nonjar");
+    assertTrue("Create non-jar file", nonJarFile.createNewFile());
+    File jarFile = new File(jarsDir, "a.jar");
+    assertTrue("Create jar file", jarFile.createNewFile());
+
+    File nofile = new File(testDir, "nofile");
+    // don't create nofile
+
+    StringBuilder cp = new StringBuilder();
+    cp.append(file.getAbsolutePath()).append(File.pathSeparator)
+      .append(dir.getAbsolutePath()).append(File.pathSeparator)
+      .append(jarsDir.getAbsolutePath() + "/*").append(File.pathSeparator)
+      .append(nofile.getAbsolutePath()).append(File.pathSeparator)
+      .append(nofile.getAbsolutePath() + "/*").append(File.pathSeparator);
+    
+    URL[] urls = constructUrlsFromClasspath(cp.toString());
+    
+    assertEquals(3, urls.length);
+    assertEquals(file.toURI().toURL(), urls[0]);
+    assertEquals(dir.toURI().toURL(), urls[1]);
+    assertEquals(jarFile.toURI().toURL(), urls[2]);
+    // nofile should be ignored
+  }
+  
+  @Test
+  public void testIsSystemClass() {
+    assertFalse(isSystemClass("org.example.Foo", null));
+    assertTrue(isSystemClass("org.example.Foo", classes("org.example.Foo")));
+    assertTrue(isSystemClass("/org.example.Foo", classes("org.example.Foo")));
+    assertTrue(isSystemClass("org.example.Foo", classes("org.example.")));
+    assertTrue(isSystemClass("net.example.Foo",
+        classes("org.example.,net.example.")));
+    assertFalse(isSystemClass("org.example.Foo",
+        classes("-org.example.Foo,org.example.")));
+    assertTrue(isSystemClass("org.example.Bar",
+        classes("-org.example.Foo.,org.example.")));
+  }
+  
+  private List<String> classes(String classes) {
+    return Lists.newArrayList(Splitter.on(',').split(classes));
+  }
+  
+  @Test
+  public void testGetResource() throws IOException {
+    URL testJar = makeTestJar().toURI().toURL();
+    
+    ClassLoader currentClassLoader = getClass().getClassLoader();
+    ClassLoader appClassloader = new ApplicationClassLoader(
+        new URL[] { testJar }, currentClassLoader, null);
+
+    assertNull("Resource should be null for current classloader",
+        currentClassLoader.getResourceAsStream("resource.txt"));
+
+    InputStream in = appClassloader.getResourceAsStream("resource.txt");
+    assertNotNull("Resource should not be null for app classloader", in);
+    assertEquals("hello", IOUtils.toString(in));
+  }
+  
+  private File makeTestJar() throws IOException {
+    File jarFile = new File(testDir, "test.jar");
+    JarOutputStream out = new JarOutputStream(new FileOutputStream(jarFile));
+    ZipEntry entry = new ZipEntry("resource.txt");
+    out.putNextEntry(entry);
+    out.write("hello".getBytes());
+    out.closeEntry();
+    out.close();
+    return jarFile;
+  }
+  
+}