HBASE-14548 Expand how table coprocessor jar and dependency path can be specified (Xiang Li)

This commit is contained in:
Jerry He 2016-07-09 18:00:41 -07:00
parent 496fd9837a
commit 632969787a
3 changed files with 95 additions and 25 deletions

View File

@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.util; package org.apache.hadoop.hbase.util;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import java.io.IOException; import java.io.IOException;
import java.net.URL; import java.net.URL;
@ -38,6 +39,8 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.IOUtils;
@ -142,7 +145,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
super(parent); super(parent);
} }
private void init(Path path, String pathPrefix, private void init(Path pathPattern, String pathPrefix,
Configuration conf) throws IOException { Configuration conf) throws IOException {
// Copy the jar to the local filesystem // Copy the jar to the local filesystem
String parentDirStr = String parentDirStr =
@ -160,7 +163,16 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
} }
} }
FileSystem fs = path.getFileSystem(conf); FileSystem fs = pathPattern.getFileSystem(conf);
Path pathPattern1 = fs.isDirectory(pathPattern) ?
new Path(pathPattern, "*.jar") : pathPattern; // append "*.jar" if a directory is specified
FileStatus[] fileStatuses = fs.globStatus(pathPattern1); // return all files that match the pattern
if (fileStatuses == null || fileStatuses.length == 0) { // if no one matches
throw new FileNotFoundException(pathPattern1.toString());
} else {
boolean validFileEncountered = false;
for (Path path : FileUtil.stat2Paths(fileStatuses)) { // for each file that match the pattern
if (fs.isFile(path)) { // only process files, skip for directories
File dst = new File(parentDirStr, "." + pathPrefix + "." File dst = new File(parentDirStr, "." + pathPrefix + "."
+ path.getName() + "." + System.currentTimeMillis() + ".jar"); + path.getName() + "." + System.currentTimeMillis() + ".jar");
fs.copyToLocalFile(path, new Path(dst.toString())); fs.copyToLocalFile(path, new Path(dst.toString()));
@ -170,7 +182,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
JarFile jarFile = new JarFile(dst.toString()); JarFile jarFile = new JarFile(dst.toString());
try { try {
Enumeration<JarEntry> entries = jarFile.entries(); Enumeration<JarEntry> entries = jarFile.entries(); // get entries inside a jar file
while (entries.hasMoreElements()) { while (entries.hasMoreElements()) {
JarEntry entry = entries.nextElement(); JarEntry entry = entries.nextElement();
Matcher m = libJarPattern.matcher(entry.getName()); Matcher m = libJarPattern.matcher(entry.getName());
@ -188,6 +200,14 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
} finally { } finally {
jarFile.close(); jarFile.close();
} }
validFileEncountered = true; // Set to true when encountering a file
}
}
if (validFileEncountered == false) { // all items returned by globStatus() are directories
throw new FileNotFoundException("No file found matching " + pathPattern1.toString());
}
}
} }
// This method is used in unit test // This method is used in unit test
@ -227,7 +247,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
return cl; return cl;
} }
if (!pathStr.endsWith(".jar")) { if (path.getFileSystem(conf).isFile(path) && !pathStr.endsWith(".jar")) {
throw new IOException(pathStr + ": not a jar file?"); throw new IOException(pathStr + ": not a jar file?");
} }

View File

@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.util;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail; import static org.junit.Assert.fail;
import java.io.File; import java.io.File;
@ -109,4 +110,46 @@ public class TestCoprocessorClassLoader {
} }
fail("Could not find the expected lib jar file"); fail("Could not find the expected lib jar file");
} }
// HBASE-14548
@Test
public void testDirectoryAndWildcard() throws Exception {
String testClassName = "TestClass";
String dataTestDir = TEST_UTIL.getDataTestDir().toString();
System.out.println(dataTestDir);
String localDirContainingJar = ClassLoaderTestHelper.localDirPath(conf);
ClassLoaderTestHelper.buildJar(dataTestDir, testClassName, null, localDirContainingJar);
ClassLoader parent = TestCoprocessorClassLoader.class.getClassLoader();
CoprocessorClassLoader.parentDirLockSet.clear(); // So that clean up can be triggered
CoprocessorClassLoader coprocessorClassLoader = null;
Path testPath = null;
// Directory
testPath = new Path(localDirContainingJar);
coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_1", conf);
verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
// Wildcard - *.jar
testPath = new Path(localDirContainingJar, "*.jar");
coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_2", conf);
verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
// Wildcard - *.j*
testPath = new Path(localDirContainingJar, "*.j*");
coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_3", conf);
verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
}
/**
* Verify the coprocessorClassLoader is not null and the expected class can be loaded successfully
* @param coprocessorClassLoader the CoprocessorClassLoader to verify
* @param className the expected class to be loaded by the coprocessorClassLoader
* @throws ClassNotFoundException
*/
private void verifyCoprocessorClassLoader(CoprocessorClassLoader coprocessorClassLoader, String className)
throws ClassNotFoundException{
assertNotNull("Classloader should be created and not null", coprocessorClassLoader);
assertEquals(className, coprocessorClassLoader.loadClass(className).getName());
}
} }

View File

@ -320,7 +320,14 @@ The value contains four pieces of information which are separated by the pipe (`
* File path: The jar file containing the Coprocessor implementation must be in a location where * File path: The jar file containing the Coprocessor implementation must be in a location where
all region servers can read it. + all region servers can read it. +
You could copy the file onto the local disk on each region server, but it is recommended to store You could copy the file onto the local disk on each region server, but it is recommended to store
it in HDFS. it in HDFS. +
https://issues.apache.org/jira/browse/HBASE-14548[HBASE-14548] allows a directory containing the jars
or some wildcards to be specified, such as: hdfs://<namenode>:<port>/user/<hadoop-user>/ or
hdfs://<namenode>:<port>/user/<hadoop-user>/*.jar. Please note that if a directory is specified,
all jar files(.jar) directly in the directory are added,
but it does not search files in the subtree rooted in the directory.
And do not contain any wildcard if you would like to specify a directory.
This enhancement applies to the ways of using the JAVA API as well.
* Class name: The full class name of the Coprocessor. * Class name: The full class name of the Coprocessor.
* Priority: An integer. The framework will determine the execution sequence of all configured * Priority: An integer. The framework will determine the execution sequence of all configured
observers registered at the same hook using priorities. This field can be left blank. In that observers registered at the same hook using priorities. This field can be left blank. In that