Apache/hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-14548 Expand how table coprocessor jar and dependency path can be specified (Xiang Li)

Browse Source
This commit is contained in:
Jerry He 2016-07-09 18:18:22 -07:00
parent 9563ab4ca1
commit fe915e10d6
2 changed files with 89 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.util;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
@ -39,6 +40,9 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.io.IOUtils;
import com.google.common.base.Preconditions;
@ -155,7 +159,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
super(parent);
}
private void init(Path path, String pathPrefix,
private void init(Path pathPattern, String pathPrefix,
Configuration conf) throws IOException {
// Copy the jar to the local filesystem
String parentDirStr =
@ -173,31 +177,50 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
}
}
FileSystem fs = path.getFileSystem(conf);
File dst = new File(parentDirStr, "." + pathPrefix + "."
+ path.getName() + "." + System.currentTimeMillis() + ".jar");
fs.copyToLocalFile(path, new Path(dst.toString()));
dst.deleteOnExit();
FileSystem fs = pathPattern.getFileSystem(conf);
Path pathPattern1 = fs.isDirectory(pathPattern) ?
new Path(pathPattern, "*.jar") : pathPattern; // append "*.jar" if a directory is specified
FileStatus[] fileStatuses = fs.globStatus(pathPattern1); // return all files that match the pattern
if (fileStatuses == null || fileStatuses.length == 0) { // if no one matches
throw new FileNotFoundException(pathPattern1.toString());
} else {
boolean validFileEncountered = false;
for (Path path : FileUtil.stat2Paths(fileStatuses)) { // for each file that match the pattern
if (fs.isFile(path)) { // only process files, skip for directories
File dst = new File(parentDirStr, "." + pathPrefix + "."
+ path.getName() + "." + System.currentTimeMillis() + ".jar");
fs.copyToLocalFile(path, new Path(dst.toString()));
dst.deleteOnExit();
addURL(dst.getCanonicalFile().toURI().toURL());
addURL(dst.getCanonicalFile().toURI().toURL());
JarFile jarFile = new JarFile(dst.toString());
try {
Enumeration<JarEntry> entries = jarFile.entries();
while (entries.hasMoreElements()) {
JarEntry entry = entries.nextElement();
Matcher m = libJarPattern.matcher(entry.getName());
if (m.matches()) {
File file = new File(parentDirStr, "." + pathPrefix + "."
+ path.getName() + "." + System.currentTimeMillis() + "." + m.group(1));
IOUtils.copyBytes(jarFile.getInputStream(entry),
new FileOutputStream(file), conf, true);
file.deleteOnExit();
addURL(file.toURI().toURL());
JarFile jarFile = new JarFile(dst.toString());
try {
Enumeration<JarEntry> entries = jarFile.entries(); // get entries inside a jar file
while (entries.hasMoreElements()) {
JarEntry entry = entries.nextElement();
Matcher m = libJarPattern.matcher(entry.getName());
if (m.matches()) {
File file = new File(parentDirStr, "." + pathPrefix + "."
+ path.getName() + "." + System.currentTimeMillis() + "." + m.group(1));
try (FileOutputStream outStream = new FileOutputStream(file)) {
IOUtils.copyBytes(jarFile.getInputStream(entry),
outStream, conf, true);
}
file.deleteOnExit();
addURL(file.toURI().toURL());
}
}
} finally {
jarFile.close();
}
validFileEncountered = true; // Set to true when encountering a file
}
}
} finally {
jarFile.close();
if (validFileEncountered == false) { // all items returned by globStatus() are directories
throw new FileNotFoundException("No file found matching " + pathPattern1.toString());
}
}
}
@ -238,7 +261,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
return cl;
}
if (!pathStr.endsWith(".jar")) {
if (path.getFileSystem(conf).isFile(path) && !pathStr.endsWith(".jar")) {
throw new IOException(pathStr + ": not a jar file?");
}

43
hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
View File

@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.util;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import java.io.File;
@ -108,4 +109,46 @@ public class TestCoprocessorClassLoader {
}
fail("Could not find the expected lib jar file");
}
// HBASE-14548
@Test
public void testDirectoryAndWildcard() throws Exception {
String testClassName = "TestClass";
String dataTestDir = TEST_UTIL.getDataTestDir().toString();
System.out.println(dataTestDir);
String localDirContainingJar = ClassLoaderTestHelper.localDirPath(conf);
ClassLoaderTestHelper.buildJar(dataTestDir, testClassName, null, localDirContainingJar);
ClassLoader parent = TestCoprocessorClassLoader.class.getClassLoader();
CoprocessorClassLoader.parentDirLockSet.clear(); // So that clean up can be triggered
CoprocessorClassLoader coprocessorClassLoader = null;
Path testPath = null;
// Directory
testPath = new Path(localDirContainingJar);
coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_1", conf);
verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
// Wildcard - *.jar
testPath = new Path(localDirContainingJar, "*.jar");
coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_2", conf);
verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
// Wildcard - *.j*
testPath = new Path(localDirContainingJar, "*.j*");
coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_3", conf);
verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
}
/**
* Verify the coprocessorClassLoader is not null and the expected class can be loaded successfully
* @param coprocessorClassLoader the CoprocessorClassLoader to verify
* @param className the expected class to be loaded by the coprocessorClassLoader
* @throws ClassNotFoundException
*/
private void verifyCoprocessorClassLoader(CoprocessorClassLoader coprocessorClassLoader, String className)
throws ClassNotFoundException{
assertNotNull("Classloader should be created and not null", coprocessorClassLoader);
assertEquals(className, coprocessorClassLoader.loadClass(className).getName());
}
}