HBASE-8327 Consolidate class loaders
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1471464 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
jxiang
parent
100cf35f5d
commit
8a3cba0cc1
|
|
@ -147,8 +147,8 @@ public class TestGet {
|
|||
}
|
||||
|
||||
Configuration conf = HBaseConfiguration.create();
|
||||
String localPath = conf.get("hbase.local.dir") + File.separator
|
||||
+ "dynamic" + File.separator + "jars" + File.separator;
|
||||
String localPath = conf.get("hbase.local.dir")
|
||||
+ File.separator + "jars" + File.separator;
|
||||
File jarFile = new File(localPath, "MockFilter.jar");
|
||||
jarFile.deleteOnExit();
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,79 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import java.net.URL;
|
||||
import java.net.URLClassLoader;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
/**
|
||||
* Base class loader that defines couple shared constants used
|
||||
* by sub-classes. It also defined method getClassLoadingLock for parallel
|
||||
* class loading and JDK 1.6 support. This method (getClassLoadingLock)
|
||||
* is similar to the same method in the base class Java ClassLoader
|
||||
* introduced in JDK 1.7, but not in JDK 1.6.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class ClassLoaderBase extends URLClassLoader {
|
||||
|
||||
// Maps class name to the corresponding lock object
|
||||
private final ConcurrentHashMap<String, Object> parallelLockMap
|
||||
= new ConcurrentHashMap<String, Object>();
|
||||
|
||||
protected static final String DEFAULT_LOCAL_DIR = "/tmp/hbase-local-dir";
|
||||
protected static final String LOCAL_DIR_KEY = "hbase.local.dir";
|
||||
|
||||
/**
|
||||
* Parent class loader.
|
||||
*/
|
||||
protected final ClassLoader parent;
|
||||
|
||||
/**
|
||||
* Creates a DynamicClassLoader that can load classes dynamically
|
||||
* from jar files under a specific folder.
|
||||
*
|
||||
* @param conf the configuration for the cluster.
|
||||
* @param parent the parent ClassLoader to set.
|
||||
*/
|
||||
public ClassLoaderBase(final ClassLoader parent) {
|
||||
super(new URL[]{}, parent);
|
||||
Preconditions.checkNotNull(parent, "No parent classloader!");
|
||||
this.parent = parent;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the lock object for class loading operations.
|
||||
*/
|
||||
protected Object getClassLoadingLock(String className) {
|
||||
Object lock = parallelLockMap.get(className);
|
||||
if (lock != null) {
|
||||
return lock;
|
||||
}
|
||||
|
||||
Object newLock = new Object();
|
||||
lock = parallelLockMap.putIfAbsent(className, newLock);
|
||||
if (lock == null) {
|
||||
lock = newLock;
|
||||
}
|
||||
return lock;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,336 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.security.AccessController;
|
||||
import java.security.PrivilegedAction;
|
||||
import java.util.Collection;
|
||||
import java.util.Enumeration;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.jar.JarEntry;
|
||||
import java.util.jar.JarFile;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.io.IOUtils;
|
||||
|
||||
import com.google.common.collect.MapMaker;
|
||||
|
||||
/**
|
||||
* ClassLoader used to load classes for Coprocessor instances.
|
||||
* <p>
|
||||
* This ClassLoader always tries to load classes from the specified coprocessor
|
||||
* jar first actually using URLClassLoader logic before delegating to the parent
|
||||
* ClassLoader, thus avoiding dependency conflicts between HBase's classpath and
|
||||
* classes in the coprocessor jar.
|
||||
* <p>
|
||||
* Certain classes are exempt from being loaded by this ClassLoader because it
|
||||
* would prevent them from being cast to the equivalent classes in the region
|
||||
* server. For example, the Coprocessor interface needs to be loaded by the
|
||||
* region server's ClassLoader to prevent a ClassCastException when casting the
|
||||
* coprocessor implementation.
|
||||
* <p>
|
||||
* A HDFS path can be used to specify the coprocessor jar. In this case, the jar
|
||||
* will be copied to local at first under some folder under ${hbase.local.dir}/jars/tmp/.
|
||||
* The local copy will be removed automatically when the HBase server instance is
|
||||
* stopped.
|
||||
* <p>
|
||||
* This ClassLoader also handles resource loading. In most cases this
|
||||
* ClassLoader will attempt to load resources from the coprocessor jar first
|
||||
* before delegating to the parent. However, like in class loading,
|
||||
* some resources need to be handled differently. For all of the Hadoop
|
||||
* default configurations (e.g. hbase-default.xml) we will check the parent
|
||||
* ClassLoader first to prevent issues such as failing the HBase default
|
||||
* configuration version check.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class CoprocessorClassLoader extends ClassLoaderBase {
|
||||
private static final Log LOG = LogFactory.getLog(CoprocessorClassLoader.class);
|
||||
|
||||
// A temporary place ${hbase.local.dir}/jars/tmp/ to store the local
|
||||
// copy of the jar file and the libraries contained in the jar.
|
||||
private static final String TMP_JARS_DIR = File.separator
|
||||
+ "jars" + File.separator + "tmp" + File.separator;
|
||||
|
||||
/**
|
||||
* External class loaders cache keyed by external jar path.
|
||||
* ClassLoader instance is stored as a weak-reference
|
||||
* to allow GC'ing when it is not used
|
||||
* (@see HBASE-7205)
|
||||
*/
|
||||
private static final ConcurrentMap<Path, CoprocessorClassLoader> classLoadersCache =
|
||||
new MapMaker().concurrencyLevel(3).weakValues().makeMap();
|
||||
|
||||
/**
|
||||
* If the class being loaded starts with any of these strings, we will skip
|
||||
* trying to load it from the coprocessor jar and instead delegate
|
||||
* directly to the parent ClassLoader.
|
||||
*/
|
||||
private static final String[] CLASS_PREFIX_EXEMPTIONS = new String[] {
|
||||
// Java standard library:
|
||||
"com.sun.",
|
||||
"launcher.",
|
||||
"java.",
|
||||
"javax.",
|
||||
"org.ietf",
|
||||
"org.omg",
|
||||
"org.w3c",
|
||||
"org.xml",
|
||||
"sunw.",
|
||||
// logging
|
||||
"org.apache.commons.logging",
|
||||
"org.apache.log4j",
|
||||
"com.hadoop",
|
||||
// Hadoop/HBase/ZK:
|
||||
"org.apache.hadoop",
|
||||
"org.apache.zookeeper",
|
||||
};
|
||||
|
||||
/**
|
||||
* If the resource being loaded matches any of these patterns, we will first
|
||||
* attempt to load the resource with the parent ClassLoader. Only if the
|
||||
* resource is not found by the parent do we attempt to load it from the coprocessor jar.
|
||||
*/
|
||||
private static final Pattern[] RESOURCE_LOAD_PARENT_FIRST_PATTERNS =
|
||||
new Pattern[] {
|
||||
Pattern.compile("^[^-]+-default\\.xml$")
|
||||
};
|
||||
|
||||
/**
|
||||
* Creates a JarClassLoader that loads classes from the given paths.
|
||||
*/
|
||||
private CoprocessorClassLoader(ClassLoader parent) {
|
||||
super(parent);
|
||||
}
|
||||
|
||||
private void init(Path path, String pathPrefix,
|
||||
Configuration conf) throws IOException {
|
||||
if (path == null) {
|
||||
throw new IOException("The jar path is null");
|
||||
}
|
||||
if (!path.toString().endsWith(".jar")) {
|
||||
throw new IOException(path.toString() + ": not a jar file?");
|
||||
}
|
||||
|
||||
// Copy the jar to the local filesystem
|
||||
String parentDirPath =
|
||||
conf.get(LOCAL_DIR_KEY, DEFAULT_LOCAL_DIR) + TMP_JARS_DIR;
|
||||
File parentDir = new File(parentDirPath);
|
||||
if (!parentDir.mkdirs() && !parentDir.isDirectory()) {
|
||||
throw new RuntimeException("Failed to create local dir " + parentDir.getPath()
|
||||
+ ", CoprocessorClassLoader failed to init");
|
||||
}
|
||||
|
||||
FileSystem fs = path.getFileSystem(conf);
|
||||
File dst = new File(parentDir, "." + pathPrefix + "."
|
||||
+ path.getName() + "." + System.currentTimeMillis() + ".jar");
|
||||
fs.copyToLocalFile(path, new Path(dst.toString()));
|
||||
dst.deleteOnExit();
|
||||
|
||||
addURL(dst.getCanonicalFile().toURI().toURL());
|
||||
|
||||
JarFile jarFile = new JarFile(dst.toString());
|
||||
try {
|
||||
Enumeration<JarEntry> entries = jarFile.entries();
|
||||
while (entries.hasMoreElements()) {
|
||||
JarEntry entry = entries.nextElement();
|
||||
if (entry.getName().matches("/lib/[^/]+\\.jar")) {
|
||||
File file = new File(parentDir, "." + pathPrefix + "." + path.getName()
|
||||
+ "." + System.currentTimeMillis() + "." + entry.getName().substring(5));
|
||||
IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true);
|
||||
file.deleteOnExit();
|
||||
addURL(file.toURI().toURL());
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
jarFile.close();
|
||||
}
|
||||
}
|
||||
|
||||
// This method is used in unit test
|
||||
public static CoprocessorClassLoader getIfCached(final Path path) {
|
||||
if (path == null) return null; // No class loader for null path
|
||||
return classLoadersCache.get(path);
|
||||
}
|
||||
|
||||
// This method is used in unit test
|
||||
public static Collection<? extends ClassLoader> getAllCached() {
|
||||
return classLoadersCache.values();
|
||||
}
|
||||
|
||||
// This method is used in unit test
|
||||
public static void clearCache() {
|
||||
classLoadersCache.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a CoprocessorClassLoader for a coprocessor jar path from cache.
|
||||
* If not in cache, create one.
|
||||
*
|
||||
* @param path the path to the coprocessor jar file to load classes from
|
||||
* @param parent the parent class loader for exempted classes
|
||||
* @param pathPrefix a prefix used in temp path name to store the jar file locally
|
||||
* @param conf the configuration used to create the class loader, if needed
|
||||
* @return a CoprocessorClassLoader for the coprocessor jar path
|
||||
* @throws IOException
|
||||
*/
|
||||
public static CoprocessorClassLoader getClassLoader(final Path path,
|
||||
final ClassLoader parent, final String pathPrefix,
|
||||
final Configuration conf) throws IOException {
|
||||
CoprocessorClassLoader cl = getIfCached(path);
|
||||
if (cl != null) {
|
||||
LOG.debug("Found classloader "+ cl + "for "+ path.toString());
|
||||
return cl;
|
||||
}
|
||||
|
||||
cl = AccessController.doPrivileged(new PrivilegedAction<CoprocessorClassLoader>() {
|
||||
@Override
|
||||
public CoprocessorClassLoader run() {
|
||||
return new CoprocessorClassLoader(parent);
|
||||
}
|
||||
});
|
||||
|
||||
cl.init(path, pathPrefix, conf);
|
||||
|
||||
// Cache class loader as a weak value, will be GC'ed when no reference left
|
||||
CoprocessorClassLoader prev = classLoadersCache.putIfAbsent(path, cl);
|
||||
if (prev != null) {
|
||||
// Lost update race, use already added class loader
|
||||
cl = prev;
|
||||
}
|
||||
return cl;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<?> loadClass(String name)
|
||||
throws ClassNotFoundException {
|
||||
// Delegate to the parent immediately if this class is exempt
|
||||
if (isClassExempt(name)) {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Skipping exempt class " + name +
|
||||
" - delegating directly to parent");
|
||||
}
|
||||
return parent.loadClass(name);
|
||||
}
|
||||
|
||||
synchronized (getClassLoadingLock(name)) {
|
||||
// Check whether the class has already been loaded:
|
||||
Class<?> clasz = findLoadedClass(name);
|
||||
if (clasz != null) {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Class " + name + " already loaded");
|
||||
}
|
||||
}
|
||||
else {
|
||||
try {
|
||||
// Try to find this class using the URLs passed to this ClassLoader
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Finding class: " + name);
|
||||
}
|
||||
clasz = findClass(name);
|
||||
} catch (ClassNotFoundException e) {
|
||||
// Class not found using this ClassLoader, so delegate to parent
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Class " + name + " not found - delegating to parent");
|
||||
}
|
||||
try {
|
||||
clasz = parent.loadClass(name);
|
||||
} catch (ClassNotFoundException e2) {
|
||||
// Class not found in this ClassLoader or in the parent ClassLoader
|
||||
// Log some debug output before re-throwing ClassNotFoundException
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Class " + name + " not found in parent loader");
|
||||
}
|
||||
throw e2;
|
||||
}
|
||||
}
|
||||
}
|
||||
return clasz;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public URL getResource(String name) {
|
||||
URL resource = null;
|
||||
boolean parentLoaded = false;
|
||||
|
||||
// Delegate to the parent first if necessary
|
||||
if (loadResourceUsingParentFirst(name)) {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Checking parent first for resource " + name);
|
||||
}
|
||||
resource = super.getResource(name);
|
||||
parentLoaded = true;
|
||||
}
|
||||
|
||||
if (resource == null) {
|
||||
synchronized (getClassLoadingLock(name)) {
|
||||
// Try to find the resource in this jar
|
||||
resource = findResource(name);
|
||||
if ((resource == null) && !parentLoaded) {
|
||||
// Not found in this jar and we haven't attempted to load
|
||||
// the resource in the parent yet; fall back to the parent
|
||||
resource = super.getResource(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
return resource;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines whether the given class should be exempt from being loaded
|
||||
* by this ClassLoader.
|
||||
* @param name the name of the class to test.
|
||||
* @return true if the class should *not* be loaded by this ClassLoader;
|
||||
* false otherwise.
|
||||
*/
|
||||
protected boolean isClassExempt(String name) {
|
||||
for (String exemptPrefix : CLASS_PREFIX_EXEMPTIONS) {
|
||||
if (name.startsWith(exemptPrefix)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines whether we should attempt to load the given resource using the
|
||||
* parent first before attempting to load the resource using this ClassLoader.
|
||||
* @param name the name of the resource to test.
|
||||
* @return true if we should attempt to load the resource using the parent
|
||||
* first; false if we should attempt to load the resource using this
|
||||
* ClassLoader first.
|
||||
*/
|
||||
protected boolean loadResourceUsingParentFirst(String name) {
|
||||
for (Pattern resourcePattern : RESOURCE_LOAD_PARENT_FIRST_PATTERNS) {
|
||||
if (resourcePattern.matcher(name).matches()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -21,7 +21,6 @@ import java.io.File;
|
|||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.net.URLClassLoader;
|
||||
import java.util.HashMap;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
|
|
@ -34,10 +33,17 @@ import org.apache.hadoop.fs.Path;
|
|||
|
||||
/**
|
||||
* This is a class loader that can load classes dynamically from new
|
||||
* jar files under a configured folder. It always uses its parent class
|
||||
* loader to load a class at first. Only if its parent class loader
|
||||
* jar files under a configured folder. The paths to the jar files are
|
||||
* converted to URLs, and URLClassLoader logic is actually used to load
|
||||
* classes. This class loader always uses its parent class loader
|
||||
* to load a class at first. Only if its parent class loader
|
||||
* can not load a class, we will try to load it using the logic here.
|
||||
* <p>
|
||||
* The configured folder can be a HDFS path. In this case, the jar files
|
||||
* under that folder will be copied to local at first under ${hbase.local.dir}/jars/.
|
||||
* The local copy will be updated if the remote copy is updated, according to its
|
||||
* last modified timestamp.
|
||||
* <p>
|
||||
* We can't unload a class already loaded. So we will use the existing
|
||||
* jar files we already know to load any class which can't be loaded
|
||||
* using the parent class loader. If we still can't load the class from
|
||||
|
|
@ -50,18 +56,15 @@ import org.apache.hadoop.fs.Path;
|
|||
* classes properly.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class DynamicClassLoader extends URLClassLoader {
|
||||
public class DynamicClassLoader extends ClassLoaderBase {
|
||||
private static final Log LOG =
|
||||
LogFactory.getLog(DynamicClassLoader.class);
|
||||
|
||||
// Dynamic jars are put under ${hbase.local.dir}/dynamic/jars/
|
||||
// Dynamic jars are put under ${hbase.local.dir}/jars/
|
||||
private static final String DYNAMIC_JARS_DIR = File.separator
|
||||
+ "dynamic" + File.separator + "jars" + File.separator;
|
||||
+ "jars" + File.separator;
|
||||
|
||||
/**
|
||||
* Parent class loader used to load any class at first.
|
||||
*/
|
||||
private final ClassLoader parent;
|
||||
private static final String DYNAMIC_JARS_DIR_KEY = "hbase.dynamic.jars.dir";
|
||||
|
||||
private File localDir;
|
||||
|
||||
|
|
@ -81,18 +84,18 @@ public class DynamicClassLoader extends URLClassLoader {
|
|||
*/
|
||||
public DynamicClassLoader(
|
||||
final Configuration conf, final ClassLoader parent) {
|
||||
super(new URL[]{}, parent);
|
||||
this.parent = parent;
|
||||
super(parent);
|
||||
|
||||
jarModifiedTime = new HashMap<String, Long>();
|
||||
String localDirPath = conf.get("hbase.local.dir") + DYNAMIC_JARS_DIR;
|
||||
String localDirPath = conf.get(
|
||||
LOCAL_DIR_KEY, DEFAULT_LOCAL_DIR) + DYNAMIC_JARS_DIR;
|
||||
localDir = new File(localDirPath);
|
||||
if (!localDir.mkdirs() && !localDir.isDirectory()) {
|
||||
throw new RuntimeException("Failed to create local dir " + localDir.getPath()
|
||||
+ ", DynamicClassLoader failed to init");
|
||||
}
|
||||
|
||||
String remotePath = conf.get("hbase.dynamic.jars.dir");
|
||||
String remotePath = conf.get(DYNAMIC_JARS_DIR_KEY);
|
||||
if (remotePath == null || remotePath.equals(localDirPath)) {
|
||||
remoteDir = null; // ignore if it is the same as the local path
|
||||
} else {
|
||||
|
|
@ -117,33 +120,35 @@ public class DynamicClassLoader extends URLClassLoader {
|
|||
LOG.debug("Class " + name + " not found - using dynamical class loader");
|
||||
}
|
||||
|
||||
// Check whether the class has already been loaded:
|
||||
Class<?> clasz = findLoadedClass(name);
|
||||
if (clasz != null) {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Class " + name + " already loaded");
|
||||
synchronized (getClassLoadingLock(name)) {
|
||||
// Check whether the class has already been loaded:
|
||||
Class<?> clasz = findLoadedClass(name);
|
||||
if (clasz != null) {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Class " + name + " already loaded");
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
try {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Finding class: " + name);
|
||||
}
|
||||
clasz = findClass(name);
|
||||
} catch (ClassNotFoundException cnfe) {
|
||||
// Load new jar files if any
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Loading new jar files, if any");
|
||||
}
|
||||
loadNewJars();
|
||||
else {
|
||||
try {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Finding class: " + name);
|
||||
}
|
||||
clasz = findClass(name);
|
||||
} catch (ClassNotFoundException cnfe) {
|
||||
// Load new jar files if any
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Loading new jar files, if any");
|
||||
}
|
||||
loadNewJars();
|
||||
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Finding class again: " + name);
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Finding class again: " + name);
|
||||
}
|
||||
clasz = findClass(name);
|
||||
}
|
||||
clasz = findClass(name);
|
||||
}
|
||||
return clasz;
|
||||
}
|
||||
return clasz;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,165 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FileWriter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.jar.JarEntry;
|
||||
import java.util.jar.JarOutputStream;
|
||||
import java.util.jar.Manifest;
|
||||
|
||||
import javax.tools.JavaCompiler;
|
||||
import javax.tools.JavaFileObject;
|
||||
import javax.tools.StandardJavaFileManager;
|
||||
import javax.tools.ToolProvider;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
|
||||
/**
|
||||
* Some utilities to help class loader testing
|
||||
*/
|
||||
public class ClassLoaderTestHelper {
|
||||
private static final Log LOG = LogFactory.getLog(ClassLoaderTestHelper.class);
|
||||
|
||||
/**
|
||||
* Jar a list of files into a jar archive.
|
||||
*
|
||||
* @param archiveFile the target jar archive
|
||||
* @param tobejared a list of files to be jared
|
||||
*/
|
||||
private static boolean createJarArchive(File archiveFile, File[] tobeJared) {
|
||||
try {
|
||||
byte buffer[] = new byte[4096];
|
||||
// Open archive file
|
||||
FileOutputStream stream = new FileOutputStream(archiveFile);
|
||||
JarOutputStream out = new JarOutputStream(stream, new Manifest());
|
||||
|
||||
for (int i = 0; i < tobeJared.length; i++) {
|
||||
if (tobeJared[i] == null || !tobeJared[i].exists()
|
||||
|| tobeJared[i].isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Add archive entry
|
||||
JarEntry jarAdd = new JarEntry(tobeJared[i].getName());
|
||||
jarAdd.setTime(tobeJared[i].lastModified());
|
||||
out.putNextEntry(jarAdd);
|
||||
|
||||
// Write file to archive
|
||||
FileInputStream in = new FileInputStream(tobeJared[i]);
|
||||
while (true) {
|
||||
int nRead = in.read(buffer, 0, buffer.length);
|
||||
if (nRead <= 0)
|
||||
break;
|
||||
out.write(buffer, 0, nRead);
|
||||
}
|
||||
in.close();
|
||||
}
|
||||
out.close();
|
||||
stream.close();
|
||||
LOG.info("Adding classes to jar file completed");
|
||||
return true;
|
||||
} catch (Exception ex) {
|
||||
LOG.error("Error: " + ex.getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a test jar for testing purpose for a given class
|
||||
* name with specified code string: save the class to a file,
|
||||
* compile it, and jar it up. If the code string passed in is
|
||||
* null, a bare empty class will be created and used.
|
||||
*
|
||||
* @param testDir the folder under which to store the test class and jar
|
||||
* @param className the test class name
|
||||
* @param code the optional test class code, which can be null.
|
||||
* If null, a bare empty class will be used
|
||||
* @return the test jar file generated
|
||||
*/
|
||||
public static File buildJar(String testDir,
|
||||
String className, String code) throws Exception {
|
||||
return buildJar(testDir, className, code, testDir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a test jar for testing purpose for a given class
|
||||
* name with specified code string.
|
||||
*
|
||||
* @param testDir the folder under which to store the test class
|
||||
* @param className the test class name
|
||||
* @param code the optional test class code, which can be null.
|
||||
* If null, an empty class will be used
|
||||
* @param folder the folder under which to store the generated jar
|
||||
* @return the test jar file generated
|
||||
*/
|
||||
public static File buildJar(String testDir,
|
||||
String className, String code, String folder) throws Exception {
|
||||
String javaCode = code != null ? code : "public class " + className + " {}";
|
||||
Path srcDir = new Path(testDir, "src");
|
||||
File srcDirPath = new File(srcDir.toString());
|
||||
srcDirPath.mkdirs();
|
||||
File sourceCodeFile = new File(srcDir.toString(), className + ".java");
|
||||
BufferedWriter bw = new BufferedWriter(new FileWriter(sourceCodeFile));
|
||||
bw.write(javaCode);
|
||||
bw.close();
|
||||
|
||||
// compile it by JavaCompiler
|
||||
JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
|
||||
ArrayList<String> srcFileNames = new ArrayList<String>();
|
||||
srcFileNames.add(sourceCodeFile.toString());
|
||||
StandardJavaFileManager fm = compiler.getStandardFileManager(null, null,
|
||||
null);
|
||||
Iterable<? extends JavaFileObject> cu =
|
||||
fm.getJavaFileObjects(sourceCodeFile);
|
||||
List<String> options = new ArrayList<String>();
|
||||
options.add("-classpath");
|
||||
// only add hbase classes to classpath. This is a little bit tricky: assume
|
||||
// the classpath is {hbaseSrc}/target/classes.
|
||||
String currentDir = new File(".").getAbsolutePath();
|
||||
String classpath = currentDir + File.separator + "target"+ File.separator
|
||||
+ "classes" + System.getProperty("path.separator")
|
||||
+ System.getProperty("java.class.path") + System.getProperty("path.separator")
|
||||
+ System.getProperty("surefire.test.class.path");
|
||||
|
||||
options.add(classpath);
|
||||
LOG.debug("Setting classpath to: " + classpath);
|
||||
|
||||
JavaCompiler.CompilationTask task = compiler.getTask(null, fm, null,
|
||||
options, null, cu);
|
||||
assertTrue("Compile file " + sourceCodeFile + " failed.", task.call());
|
||||
|
||||
// build a jar file by the classes files
|
||||
String jarFileName = className + ".jar";
|
||||
File jarFile = new File(folder, jarFileName);
|
||||
if (!createJarArchive(jarFile,
|
||||
new File[]{new File(srcDir.toString(), className + ".class")})){
|
||||
assertTrue("Build jar file failed.", false);
|
||||
}
|
||||
return jarFile;
|
||||
}
|
||||
}
|
||||
|
|
@ -18,29 +18,14 @@
|
|||
*/
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FileWriter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.jar.JarEntry;
|
||||
import java.util.jar.JarOutputStream;
|
||||
import java.util.jar.Manifest;
|
||||
|
||||
import javax.tools.JavaCompiler;
|
||||
import javax.tools.JavaFileObject;
|
||||
import javax.tools.StandardJavaFileManager;
|
||||
import javax.tools.ToolProvider;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
import org.apache.hadoop.hbase.SmallTests;
|
||||
|
|
@ -62,95 +47,13 @@ public class TestDynamicClassLoader {
|
|||
conf.set("hbase.dynamic.jars.dir", TEST_UTIL.getDataTestDir().toString());
|
||||
}
|
||||
|
||||
// generate jar file
|
||||
private boolean createJarArchive(File archiveFile, File[] tobeJared) {
|
||||
try {
|
||||
byte buffer[] = new byte[4096];
|
||||
// Open archive file
|
||||
FileOutputStream stream = new FileOutputStream(archiveFile);
|
||||
JarOutputStream out = new JarOutputStream(stream, new Manifest());
|
||||
|
||||
for (int i = 0; i < tobeJared.length; i++) {
|
||||
if (tobeJared[i] == null || !tobeJared[i].exists()
|
||||
|| tobeJared[i].isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Add archive entry
|
||||
JarEntry jarAdd = new JarEntry(tobeJared[i].getName());
|
||||
jarAdd.setTime(tobeJared[i].lastModified());
|
||||
out.putNextEntry(jarAdd);
|
||||
|
||||
// Write file to archive
|
||||
FileInputStream in = new FileInputStream(tobeJared[i]);
|
||||
while (true) {
|
||||
int nRead = in.read(buffer, 0, buffer.length);
|
||||
if (nRead <= 0)
|
||||
break;
|
||||
out.write(buffer, 0, nRead);
|
||||
}
|
||||
in.close();
|
||||
}
|
||||
out.close();
|
||||
stream.close();
|
||||
LOG.info("Adding classes to jar file completed");
|
||||
return true;
|
||||
} catch (Exception ex) {
|
||||
LOG.error("Error: " + ex.getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private File buildJar(
|
||||
String className, String folder) throws Exception {
|
||||
String javaCode = "public class " + className + " {}";
|
||||
Path srcDir = new Path(TEST_UTIL.getDataTestDir(), "src");
|
||||
File srcDirPath = new File(srcDir.toString());
|
||||
srcDirPath.mkdirs();
|
||||
File sourceCodeFile = new File(srcDir.toString(), className + ".java");
|
||||
BufferedWriter bw = new BufferedWriter(new FileWriter(sourceCodeFile));
|
||||
bw.write(javaCode);
|
||||
bw.close();
|
||||
|
||||
// compile it by JavaCompiler
|
||||
JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
|
||||
ArrayList<String> srcFileNames = new ArrayList<String>();
|
||||
srcFileNames.add(sourceCodeFile.toString());
|
||||
StandardJavaFileManager fm = compiler.getStandardFileManager(null, null,
|
||||
null);
|
||||
Iterable<? extends JavaFileObject> cu =
|
||||
fm.getJavaFileObjects(sourceCodeFile);
|
||||
List<String> options = new ArrayList<String>();
|
||||
options.add("-classpath");
|
||||
// only add hbase classes to classpath. This is a little bit tricky: assume
|
||||
// the classpath is {hbaseSrc}/target/classes.
|
||||
String currentDir = new File(".").getAbsolutePath();
|
||||
String classpath =
|
||||
currentDir + File.separator + "target"+ File.separator + "classes" +
|
||||
System.getProperty("path.separator") + System.getProperty("java.class.path");
|
||||
options.add(classpath);
|
||||
LOG.debug("Setting classpath to: "+classpath);
|
||||
|
||||
JavaCompiler.CompilationTask task = compiler.getTask(null, fm, null,
|
||||
options, null, cu);
|
||||
assertTrue("Compile file " + sourceCodeFile + " failed.", task.call());
|
||||
|
||||
// build a jar file by the classes files
|
||||
String jarFileName = className + ".jar";
|
||||
File jarFile = new File(folder, jarFileName);
|
||||
if (!createJarArchive(jarFile,
|
||||
new File[]{new File(srcDir.toString(), className + ".class")})){
|
||||
assertTrue("Build jar file failed.", false);
|
||||
}
|
||||
return jarFile;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLoadClassFromLocalPath() throws Exception {
|
||||
ClassLoader parent = TestDynamicClassLoader.class.getClassLoader();
|
||||
DynamicClassLoader classLoader = new DynamicClassLoader(conf, parent);
|
||||
|
||||
String className = "TestLoadClassFromLocalPath";
|
||||
deleteClass(className);
|
||||
try {
|
||||
classLoader.loadClass(className);
|
||||
fail("Should not be able to load class " + className);
|
||||
|
|
@ -159,13 +62,12 @@ public class TestDynamicClassLoader {
|
|||
}
|
||||
|
||||
try {
|
||||
buildJar(className, localDirPath());
|
||||
String folder = TEST_UTIL.getDataTestDir().toString();
|
||||
ClassLoaderTestHelper.buildJar(folder, className, null, localDirPath());
|
||||
classLoader.loadClass(className);
|
||||
} catch (ClassNotFoundException cnfe) {
|
||||
LOG.error("Should be able to load class " + className, cnfe);
|
||||
fail(cnfe.getMessage());
|
||||
} finally {
|
||||
deleteClass(className);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -175,6 +77,7 @@ public class TestDynamicClassLoader {
|
|||
DynamicClassLoader classLoader = new DynamicClassLoader(conf, parent);
|
||||
|
||||
String className = "TestLoadClassFromAnotherPath";
|
||||
deleteClass(className);
|
||||
try {
|
||||
classLoader.loadClass(className);
|
||||
fail("Should not be able to load class " + className);
|
||||
|
|
@ -183,30 +86,32 @@ public class TestDynamicClassLoader {
|
|||
}
|
||||
|
||||
try {
|
||||
buildJar(className, TEST_UTIL.getDataTestDir().toString());
|
||||
String folder = TEST_UTIL.getDataTestDir().toString();
|
||||
ClassLoaderTestHelper.buildJar(folder, className, null);
|
||||
classLoader.loadClass(className);
|
||||
} catch (ClassNotFoundException cnfe) {
|
||||
LOG.error("Should be able to load class " + className, cnfe);
|
||||
fail(cnfe.getMessage());
|
||||
} finally {
|
||||
deleteClass(className);
|
||||
}
|
||||
}
|
||||
|
||||
private String localDirPath() {
|
||||
return conf.get("hbase.local.dir") + File.separator
|
||||
+ "dynamic" + File.separator + "jars" + File.separator;
|
||||
return conf.get("hbase.local.dir")
|
||||
+ File.separator + "jars" + File.separator;
|
||||
}
|
||||
|
||||
private void deleteClass(String className) throws Exception {
|
||||
String jarFileName = className + ".jar";
|
||||
File file = new File(TEST_UTIL.getDataTestDir().toString(), jarFileName);
|
||||
file.deleteOnExit();
|
||||
file.delete();
|
||||
assertFalse("Should be deleted: " + file.getPath(), file.exists());
|
||||
|
||||
file = new File(conf.get("hbase.dynamic.jars.dir"), jarFileName);
|
||||
file.deleteOnExit();
|
||||
file.delete();
|
||||
assertFalse("Should be deleted: " + file.getPath(), file.exists());
|
||||
|
||||
file = new File(localDirPath(), jarFileName);
|
||||
file.deleteOnExit();
|
||||
file.delete();
|
||||
assertFalse("Should be deleted: " + file.getPath(), file.exists());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,213 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.coprocessor;
|
||||
|
||||
import java.net.URL;
|
||||
import java.net.URLClassLoader;
|
||||
import java.util.List;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
/**
|
||||
* ClassLoader used to load Coprocessor instances.
|
||||
*
|
||||
* This ClassLoader always tries to load classes from the Coprocessor jar first
|
||||
* before delegating to the parent ClassLoader, thus avoiding dependency
|
||||
* conflicts between HBase's classpath and classes in the coprocessor's jar.
|
||||
* Certain classes are exempt from being loaded by this ClassLoader because it
|
||||
* would prevent them from being cast to the equivalent classes in the region
|
||||
* server. For example, the Coprocessor interface needs to be loaded by the
|
||||
* region server's ClassLoader to prevent a ClassCastException when casting the
|
||||
* coprocessor implementation.
|
||||
*
|
||||
* This ClassLoader also handles resource loading. In most cases this
|
||||
* ClassLoader will attempt to load resources from the coprocessor jar first
|
||||
* before delegating to the parent. However, like in class loading,
|
||||
* some resources need to be handled differently. For all of the Hadoop
|
||||
* default configurations (e.g. hbase-default.xml) we will check the parent
|
||||
* ClassLoader first to prevent issues such as failing the HBase default
|
||||
* configuration version check.
|
||||
*/
|
||||
public class CoprocessorClassLoader extends URLClassLoader {
|
||||
private static final Log LOG =
|
||||
LogFactory.getLog(CoprocessorClassLoader.class);
|
||||
|
||||
/**
|
||||
* If the class being loaded starts with any of these strings, we will skip
|
||||
* trying to load it from the coprocessor jar and instead delegate
|
||||
* directly to the parent ClassLoader.
|
||||
*/
|
||||
private static final String[] CLASS_PREFIX_EXEMPTIONS = new String[] {
|
||||
// Java standard library:
|
||||
"com.sun.",
|
||||
"launcher.",
|
||||
"java.",
|
||||
"javax.",
|
||||
"org.ietf",
|
||||
"org.omg",
|
||||
"org.w3c",
|
||||
"org.xml",
|
||||
"sunw.",
|
||||
// logging
|
||||
"org.apache.commons.logging",
|
||||
"org.apache.log4j",
|
||||
"com.hadoop",
|
||||
// Hadoop/HBase/ZK:
|
||||
"org.apache.hadoop",
|
||||
"org.apache.zookeeper",
|
||||
};
|
||||
|
||||
/**
|
||||
* If the resource being loaded matches any of these patterns, we will first
|
||||
* attempt to load the resource with the parent ClassLoader. Only if the
|
||||
* resource is not found by the parent do we attempt to load it from the
|
||||
* coprocessor jar.
|
||||
*/
|
||||
private static final Pattern[] RESOURCE_LOAD_PARENT_FIRST_PATTERNS =
|
||||
new Pattern[] {
|
||||
Pattern.compile("^[^-]+-default\\.xml$")
|
||||
};
|
||||
|
||||
/**
|
||||
* Parent classloader used to load any class not matching the exemption list.
|
||||
*/
|
||||
private final ClassLoader parent;
|
||||
|
||||
/**
|
||||
* Creates a CoprocessorClassLoader that loads classes from the given paths.
|
||||
* @param paths paths from which to load classes.
|
||||
* @param parent the parent ClassLoader to set.
|
||||
*/
|
||||
public CoprocessorClassLoader(List<URL> paths, ClassLoader parent) {
|
||||
super(paths.toArray(new URL[]{}), parent);
|
||||
this.parent = parent;
|
||||
if (parent == null) {
|
||||
throw new IllegalArgumentException("No parent classloader!");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
synchronized public Class<?> loadClass(String name)
|
||||
throws ClassNotFoundException {
|
||||
// Delegate to the parent immediately if this class is exempt
|
||||
if (isClassExempt(name)) {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Skipping exempt class " + name +
|
||||
" - delegating directly to parent");
|
||||
}
|
||||
return parent.loadClass(name);
|
||||
}
|
||||
|
||||
// Check whether the class has already been loaded:
|
||||
Class<?> clasz = findLoadedClass(name);
|
||||
if (clasz != null) {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Class " + name + " already loaded");
|
||||
}
|
||||
}
|
||||
else {
|
||||
try {
|
||||
// Try to find this class using the URLs passed to this ClassLoader,
|
||||
// which includes the coprocessor jar
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Finding class: " + name);
|
||||
}
|
||||
clasz = findClass(name);
|
||||
} catch (ClassNotFoundException e) {
|
||||
// Class not found using this ClassLoader, so delegate to parent
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Class " + name + " not found - delegating to parent");
|
||||
}
|
||||
try {
|
||||
clasz = parent.loadClass(name);
|
||||
} catch (ClassNotFoundException e2) {
|
||||
// Class not found in this ClassLoader or in the parent ClassLoader
|
||||
// Log some debug output before rethrowing ClassNotFoundException
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Class " + name + " not found in parent loader");
|
||||
}
|
||||
throw e2;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return clasz;
|
||||
}
|
||||
|
||||
@Override
|
||||
synchronized public URL getResource(String name) {
|
||||
URL resource = null;
|
||||
boolean parentLoaded = false;
|
||||
|
||||
// Delegate to the parent first if necessary
|
||||
if (loadResourceUsingParentFirst(name)) {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Checking parent first for resource " + name);
|
||||
}
|
||||
resource = super.getResource(name);
|
||||
parentLoaded = true;
|
||||
}
|
||||
|
||||
if (resource == null) {
|
||||
// Try to find the resource in the coprocessor jar
|
||||
resource = findResource(name);
|
||||
if ((resource == null) && !parentLoaded) {
|
||||
// Not found in the coprocessor jar and we haven't attempted to load
|
||||
// the resource in the parent yet; fall back to the parent
|
||||
resource = super.getResource(name);
|
||||
}
|
||||
}
|
||||
|
||||
return resource;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines whether the given class should be exempt from being loaded
|
||||
* by this ClassLoader.
|
||||
* @param name the name of the class to test.
|
||||
* @return true if the class should *not* be loaded by this ClassLoader;
|
||||
* false otherwise.
|
||||
*/
|
||||
protected boolean isClassExempt(String name) {
|
||||
for (String exemptPrefix : CLASS_PREFIX_EXEMPTIONS) {
|
||||
if (name.startsWith(exemptPrefix)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines whether we should attempt to load the given resource using the
|
||||
* parent first before attempting to load the resource using this ClassLoader.
|
||||
* @param name the name of the resource to test.
|
||||
* @return true if we should attempt to load the resource using the parent
|
||||
* first; false if we should attempt to load the resource using this
|
||||
* ClassLoader first.
|
||||
*/
|
||||
protected boolean loadResourceUsingParentFirst(String name) {
|
||||
for (Pattern resourcePattern : RESOURCE_LOAD_PARENT_FIRST_PATTERNS) {
|
||||
if (resourcePattern.matcher(name).matches()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -19,39 +19,51 @@
|
|||
|
||||
package org.apache.hadoop.hbase.coprocessor;
|
||||
|
||||
import com.google.common.collect.MapMaker;
|
||||
import com.google.protobuf.Service;
|
||||
import com.google.protobuf.ServiceException;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.SortedSet;
|
||||
import java.util.TreeSet;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.Coprocessor;
|
||||
import org.apache.hadoop.hbase.CoprocessorEnvironment;
|
||||
import org.apache.hadoop.hbase.exceptions.DoNotRetryIOException;
|
||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||
import org.apache.hadoop.hbase.client.*;
|
||||
import org.apache.hadoop.hbase.Server;
|
||||
import org.apache.hadoop.hbase.client.Append;
|
||||
import org.apache.hadoop.hbase.client.Delete;
|
||||
import org.apache.hadoop.hbase.client.Durability;
|
||||
import org.apache.hadoop.hbase.client.Get;
|
||||
import org.apache.hadoop.hbase.client.HTable;
|
||||
import org.apache.hadoop.hbase.client.HTableInterface;
|
||||
import org.apache.hadoop.hbase.client.Increment;
|
||||
import org.apache.hadoop.hbase.client.Put;
|
||||
import org.apache.hadoop.hbase.client.Result;
|
||||
import org.apache.hadoop.hbase.client.ResultScanner;
|
||||
import org.apache.hadoop.hbase.client.Row;
|
||||
import org.apache.hadoop.hbase.client.RowMutations;
|
||||
import org.apache.hadoop.hbase.client.Scan;
|
||||
import org.apache.hadoop.hbase.client.coprocessor.Batch;
|
||||
import org.apache.hadoop.hbase.exceptions.DoNotRetryIOException;
|
||||
import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
|
||||
import org.apache.hadoop.hbase.util.SortedCopyOnWriteSet;
|
||||
import org.apache.hadoop.hbase.util.VersionInfo;
|
||||
import org.apache.hadoop.hbase.Server;
|
||||
import org.apache.hadoop.io.IOUtils;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.security.AccessController;
|
||||
import java.security.PrivilegedAction;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentMap;
|
||||
import java.util.jar.JarEntry;
|
||||
import java.util.jar.JarFile;
|
||||
import com.google.protobuf.Service;
|
||||
import com.google.protobuf.ServiceException;
|
||||
|
||||
/**
|
||||
* Provides the common setup framework and runtime services for coprocessor
|
||||
|
|
@ -73,10 +85,6 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
|
|||
public static final String WAL_COPROCESSOR_CONF_KEY =
|
||||
"hbase.coprocessor.wal.classes";
|
||||
|
||||
//coprocessor jars are put under ${hbase.local.dir}/coprocessor/jars/
|
||||
private static final String COPROCESSOR_JARS_DIR = File.separator
|
||||
+ "coprocessor" + File.separator + "jars" + File.separator;
|
||||
|
||||
private static final Log LOG = LogFactory.getLog(CoprocessorHost.class);
|
||||
/** Ordered set of loaded coprocessors with lock */
|
||||
protected SortedSet<E> coprocessors =
|
||||
|
|
@ -86,15 +94,6 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
|
|||
protected String pathPrefix;
|
||||
protected volatile int loadSequence;
|
||||
|
||||
/*
|
||||
* External classloaders cache keyed by external jar path.
|
||||
* ClassLoader instance is stored as a weak-reference
|
||||
* to allow GC'ing when no CoprocessorHost is using it
|
||||
* (@see HBASE-7205)
|
||||
*/
|
||||
static ConcurrentMap<Path, ClassLoader> classLoadersCache =
|
||||
new MapMaker().concurrencyLevel(3).weakValues().makeMap();
|
||||
|
||||
public CoprocessorHost() {
|
||||
pathPrefix = UUID.randomUUID().toString();
|
||||
}
|
||||
|
|
@ -175,7 +174,6 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
|
|||
* @param conf configuration for coprocessor
|
||||
* @throws java.io.IOException Exception
|
||||
*/
|
||||
@SuppressWarnings("deprecation")
|
||||
public E load(Path path, String className, int priority,
|
||||
Configuration conf) throws IOException {
|
||||
Class<?> implClass = null;
|
||||
|
|
@ -190,81 +188,8 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
|
|||
throw new IOException("No jar path specified for " + className);
|
||||
}
|
||||
} else {
|
||||
// Have we already loaded the class, perhaps from an earlier region open
|
||||
// for the same table?
|
||||
cl = classLoadersCache.get(path);
|
||||
if (cl != null){
|
||||
LOG.debug("Found classloader "+ cl + "for "+path.toString());
|
||||
try {
|
||||
implClass = cl.loadClass(className);
|
||||
} catch (ClassNotFoundException e) {
|
||||
LOG.info("Class " + className + " needs to be loaded from a file - " +
|
||||
path + ".");
|
||||
// go ahead to load from file system.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If not, load
|
||||
if (implClass == null) {
|
||||
if (path == null) {
|
||||
throw new IOException("No jar path specified for " + className);
|
||||
}
|
||||
// copy the jar to the local filesystem
|
||||
if (!path.toString().endsWith(".jar")) {
|
||||
throw new IOException(path.toString() + ": not a jar file?");
|
||||
}
|
||||
FileSystem fs = path.getFileSystem(this.conf);
|
||||
File parentDir = new File(this.conf.get("hbase.local.dir") + COPROCESSOR_JARS_DIR);
|
||||
parentDir.mkdirs();
|
||||
File dst = new File(parentDir, "." + pathPrefix +
|
||||
"." + className + "." + System.currentTimeMillis() + ".jar");
|
||||
fs.copyToLocalFile(path, new Path(dst.toString()));
|
||||
dst.deleteOnExit();
|
||||
|
||||
// TODO: code weaving goes here
|
||||
|
||||
// TODO: wrap heap allocations and enforce maximum usage limits
|
||||
|
||||
/* TODO: inject code into loop headers that monitors CPU use and
|
||||
aborts runaway user code */
|
||||
|
||||
// load the jar and get the implementation main class
|
||||
// NOTE: Path.toURL is deprecated (toURI instead) but the URLClassLoader
|
||||
// unsurprisingly wants URLs, not URIs; so we will use the deprecated
|
||||
// method which returns URLs for as long as it is available
|
||||
final List<URL> paths = new ArrayList<URL>();
|
||||
URL url = dst.getCanonicalFile().toURL();
|
||||
paths.add(url);
|
||||
|
||||
JarFile jarFile = new JarFile(dst.toString());
|
||||
Enumeration<JarEntry> entries = jarFile.entries();
|
||||
while (entries.hasMoreElements()) {
|
||||
JarEntry entry = entries.nextElement();
|
||||
if (entry.getName().matches("/lib/[^/]+\\.jar")) {
|
||||
File file = new File(parentDir, "." + pathPrefix +
|
||||
"." + className + "." + System.currentTimeMillis() + "." + entry.getName().substring(5));
|
||||
IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true);
|
||||
file.deleteOnExit();
|
||||
paths.add(file.toURL());
|
||||
}
|
||||
}
|
||||
jarFile.close();
|
||||
|
||||
cl = AccessController.doPrivileged(new PrivilegedAction<CoprocessorClassLoader>() {
|
||||
@Override
|
||||
public CoprocessorClassLoader run() {
|
||||
return new CoprocessorClassLoader(paths, this.getClass().getClassLoader());
|
||||
}
|
||||
});
|
||||
|
||||
// cache cp classloader as a weak value, will be GC'ed when no reference left
|
||||
ClassLoader prev = classLoadersCache.putIfAbsent(path, cl);
|
||||
if (prev != null) {
|
||||
//lost update race, use already added classloader
|
||||
cl = prev;
|
||||
}
|
||||
|
||||
cl = CoprocessorClassLoader.getClassLoader(
|
||||
path, getClass().getClassLoader(), pathPrefix, conf);
|
||||
try {
|
||||
implClass = cl.loadClass(className);
|
||||
} catch (ClassNotFoundException e) {
|
||||
|
|
|
|||
|
|
@ -26,13 +26,14 @@ import org.apache.hadoop.hbase.*;
|
|||
import org.apache.hadoop.hbase.client.HBaseAdmin;
|
||||
import org.apache.hadoop.hbase.regionserver.HRegion;
|
||||
import org.apache.hadoop.hbase.regionserver.TestServerCustomProtocol;
|
||||
import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
|
||||
import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.ServerLoad;
|
||||
import org.apache.hadoop.hbase.RegionLoad;
|
||||
|
||||
import javax.tools.*;
|
||||
import java.io.*;
|
||||
import java.util.*;
|
||||
import java.util.jar.*;
|
||||
|
|
@ -41,6 +42,7 @@ import org.junit.*;
|
|||
import org.junit.experimental.categories.Category;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
|
||||
|
|
@ -103,90 +105,11 @@ public class TestClassLoading {
|
|||
TEST_UTIL.shutdownMiniCluster();
|
||||
}
|
||||
|
||||
// generate jar file
|
||||
private boolean createJarArchive(File archiveFile, File[] tobeJared) {
|
||||
try {
|
||||
byte buffer[] = new byte[BUFFER_SIZE];
|
||||
// Open archive file
|
||||
FileOutputStream stream = new FileOutputStream(archiveFile);
|
||||
JarOutputStream out = new JarOutputStream(stream, new Manifest());
|
||||
|
||||
for (int i = 0; i < tobeJared.length; i++) {
|
||||
if (tobeJared[i] == null || !tobeJared[i].exists()
|
||||
|| tobeJared[i].isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Add archive entry
|
||||
JarEntry jarAdd = new JarEntry(tobeJared[i].getName());
|
||||
jarAdd.setTime(tobeJared[i].lastModified());
|
||||
out.putNextEntry(jarAdd);
|
||||
|
||||
// Write file to archive
|
||||
FileInputStream in = new FileInputStream(tobeJared[i]);
|
||||
while (true) {
|
||||
int nRead = in.read(buffer, 0, buffer.length);
|
||||
if (nRead <= 0)
|
||||
break;
|
||||
out.write(buffer, 0, nRead);
|
||||
}
|
||||
in.close();
|
||||
}
|
||||
out.close();
|
||||
stream.close();
|
||||
LOG.info("Adding classes to jar file completed");
|
||||
return true;
|
||||
} catch (Exception ex) {
|
||||
LOG.error("Error: " + ex.getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private File buildCoprocessorJar(String className) throws Exception {
|
||||
// compose a java source file.
|
||||
String javaCode = "import org.apache.hadoop.hbase.coprocessor.*;" +
|
||||
static File buildCoprocessorJar(String className) throws Exception {
|
||||
String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
|
||||
"public class " + className + " extends BaseRegionObserver {}";
|
||||
Path baseDir = TEST_UTIL.getDataTestDir();
|
||||
Path srcDir = new Path(TEST_UTIL.getDataTestDir(), "src");
|
||||
File srcDirPath = new File(srcDir.toString());
|
||||
srcDirPath.mkdirs();
|
||||
File sourceCodeFile = new File(srcDir.toString(), className + ".java");
|
||||
BufferedWriter bw = new BufferedWriter(new FileWriter(sourceCodeFile));
|
||||
bw.write(javaCode);
|
||||
bw.close();
|
||||
|
||||
// compile it by JavaCompiler
|
||||
JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
|
||||
ArrayList<String> srcFileNames = new ArrayList<String>();
|
||||
srcFileNames.add(sourceCodeFile.toString());
|
||||
StandardJavaFileManager fm = compiler.getStandardFileManager(null, null,
|
||||
null);
|
||||
Iterable<? extends JavaFileObject> cu =
|
||||
fm.getJavaFileObjects(sourceCodeFile);
|
||||
List<String> options = new ArrayList<String>();
|
||||
options.add("-classpath");
|
||||
// only add hbase classes to classpath. This is a little bit tricky: assume
|
||||
// the classpath is {hbaseSrc}/target/classes.
|
||||
String currentDir = new File(".").getAbsolutePath();
|
||||
String classpath =
|
||||
currentDir + File.separator + "target"+ File.separator + "classes" +
|
||||
System.getProperty("path.separator") + System.getProperty("java.class.path");
|
||||
options.add(classpath);
|
||||
LOG.debug("Setting classpath to: "+classpath);
|
||||
|
||||
JavaCompiler.CompilationTask task = compiler.getTask(null, fm, null,
|
||||
options, null, cu);
|
||||
assertTrue("Compile file " + sourceCodeFile + " failed.", task.call());
|
||||
|
||||
// build a jar file by the classes files
|
||||
String jarFileName = className + ".jar";
|
||||
File jarFile = new File(baseDir.toString(), jarFileName);
|
||||
if (!createJarArchive(jarFile,
|
||||
new File[]{new File(srcDir.toString(), className + ".class")})){
|
||||
assertTrue("Build jar file failed.", false);
|
||||
}
|
||||
|
||||
return jarFile;
|
||||
return ClassLoaderTestHelper.buildJar(
|
||||
TEST_UTIL.getDataTestDir().toString(), className, code);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -235,7 +158,7 @@ public class TestClassLoading {
|
|||
}
|
||||
admin.deleteTable(tableName);
|
||||
}
|
||||
CoprocessorHost.classLoadersCache.clear();
|
||||
CoprocessorClassLoader.clearCache();
|
||||
byte[] startKey = {10, 63};
|
||||
byte[] endKey = {12, 43};
|
||||
admin.createTable(htd, startKey, endKey, 4);
|
||||
|
|
@ -282,22 +205,22 @@ public class TestClassLoading {
|
|||
assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
|
||||
assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
|
||||
// check if CP classloaders are cached
|
||||
assertTrue(jarFileOnHDFS1 + " was not cached",
|
||||
CoprocessorHost.classLoadersCache.containsKey(pathOnHDFS1));
|
||||
assertTrue(jarFileOnHDFS2 + " was not cached",
|
||||
CoprocessorHost.classLoadersCache.containsKey(pathOnHDFS2));
|
||||
assertNotNull(jarFileOnHDFS1 + " was not cached",
|
||||
CoprocessorClassLoader.getIfCached(pathOnHDFS1));
|
||||
assertNotNull(jarFileOnHDFS2 + " was not cached",
|
||||
CoprocessorClassLoader.getIfCached(pathOnHDFS2));
|
||||
//two external jar used, should be one classloader per jar
|
||||
assertEquals("The number of cached classloaders should be equal to the number" +
|
||||
" of external jar files",
|
||||
2, CoprocessorHost.classLoadersCache.size());
|
||||
2, CoprocessorClassLoader.getAllCached().size());
|
||||
//check if region active classloaders are shared across all RS regions
|
||||
Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
|
||||
CoprocessorHost.classLoadersCache.values());
|
||||
CoprocessorClassLoader.getAllCached());
|
||||
for (Map.Entry<HRegion, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
|
||||
assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
|
||||
+ " ClassLoader Cache:" + externalClassLoaders
|
||||
+ " Region ClassLoaders:" + regionCP.getValue(),
|
||||
externalClassLoaders.containsAll(regionCP.getValue()));
|
||||
+ " ClassLoader Cache:" + externalClassLoaders
|
||||
+ " Region ClassLoaders:" + regionCP.getValue(),
|
||||
externalClassLoaders.containsAll(regionCP.getValue()));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue