HBASE-6824. Introduce ${hbase.local.dir} and save coprocessor jars there
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1431081 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Enis Soztutar
parent
3decb8b16e
commit
fd34e19f8e
|
|
@ -72,6 +72,10 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
|
|||
public static final String WAL_COPROCESSOR_CONF_KEY =
|
||||
"hbase.coprocessor.wal.classes";
|
||||
|
||||
//coprocessor jars are put under ${hbase.local.dir}/coprocessor/jars/
|
||||
private static final String COPROCESSOR_JARS_DIR = File.separator
|
||||
+ "coprocessor" + File.separator + "jars" + File.separator;
|
||||
|
||||
private static final Log LOG = LogFactory.getLog(CoprocessorHost.class);
|
||||
/** Ordered set of loaded coprocessors with lock */
|
||||
protected SortedSet<E> coprocessors =
|
||||
|
|
@ -209,13 +213,13 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
|
|||
if (!path.toString().endsWith(".jar")) {
|
||||
throw new IOException(path.toString() + ": not a jar file?");
|
||||
}
|
||||
FileSystem fs = path.getFileSystem(HBaseConfiguration.create());
|
||||
Path dst = new Path(System.getProperty("java.io.tmpdir") +
|
||||
java.io.File.separator +"." + pathPrefix +
|
||||
FileSystem fs = path.getFileSystem(this.conf);
|
||||
File parentDir = new File(this.conf.get("hbase.local.dir") + COPROCESSOR_JARS_DIR);
|
||||
parentDir.mkdirs();
|
||||
File dst = new File(parentDir, "." + pathPrefix +
|
||||
"." + className + "." + System.currentTimeMillis() + ".jar");
|
||||
fs.copyToLocalFile(path, dst);
|
||||
File tmpLocal = new File(dst.toString());
|
||||
tmpLocal.deleteOnExit();
|
||||
fs.copyToLocalFile(path, new Path(dst.toString()));
|
||||
dst.deleteOnExit();
|
||||
|
||||
// TODO: code weaving goes here
|
||||
|
||||
|
|
@ -229,7 +233,7 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
|
|||
// unsurprisingly wants URLs, not URIs; so we will use the deprecated
|
||||
// method which returns URLs for as long as it is available
|
||||
List<URL> paths = new ArrayList<URL>();
|
||||
URL url = new File(dst.toString()).getCanonicalFile().toURL();
|
||||
URL url = dst.getCanonicalFile().toURL();
|
||||
paths.add(url);
|
||||
|
||||
JarFile jarFile = new JarFile(dst.toString());
|
||||
|
|
@ -237,8 +241,7 @@ public abstract class CoprocessorHost<E extends CoprocessorEnvironment> {
|
|||
while (entries.hasMoreElements()) {
|
||||
JarEntry entry = entries.nextElement();
|
||||
if (entry.getName().matches("/lib/[^/]+\\.jar")) {
|
||||
File file = new File(System.getProperty("java.io.tmpdir") +
|
||||
java.io.File.separator +"." + pathPrefix +
|
||||
File file = new File(parentDir, "." + pathPrefix +
|
||||
"." + className + "." + System.currentTimeMillis() + "." + entry.getName().substring(5));
|
||||
IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true);
|
||||
file.deleteOnExit();
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ public class MasterCoprocessorHost
|
|||
private MasterServices masterServices;
|
||||
|
||||
MasterCoprocessorHost(final MasterServices services, final Configuration conf) {
|
||||
this.conf = conf;
|
||||
this.masterServices = services;
|
||||
loadSystemCoprocessors(conf, MASTER_COPROCESSOR_CONF_KEY);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -134,6 +134,7 @@ public class RegionCoprocessorHost
|
|||
*/
|
||||
public RegionCoprocessorHost(final HRegion region,
|
||||
final RegionServerServices rsServices, final Configuration conf) {
|
||||
this.conf = conf;
|
||||
this.rsServices = rsServices;
|
||||
this.region = region;
|
||||
this.pathPrefix = Integer.toString(this.region.getRegionInfo().hashCode());
|
||||
|
|
|
|||
|
|
@ -49,13 +49,20 @@
|
|||
</property>
|
||||
<property>
|
||||
<name>hbase.tmp.dir</name>
|
||||
<value>/tmp/hbase-${user.name}</value>
|
||||
<value>${java.io.tmpdir}/hbase-${user.name}</value>
|
||||
<description>Temporary directory on the local filesystem.
|
||||
Change this setting to point to a location more permanent
|
||||
than '/tmp' (The '/tmp' directory is often cleared on
|
||||
machine restart).
|
||||
</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>hbase.local.dir</name>
|
||||
<value>${hbase.tmp.dir}/local/</value>
|
||||
<description>Directory on the local filesystem to be used
|
||||
as a local storage.
|
||||
</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>hbase.master.info.port</name>
|
||||
<value>60010</value>
|
||||
|
|
|
|||
|
|
@ -266,6 +266,9 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
|
|||
"mapred.local.dir",
|
||||
testPath, "mapred-local-dir");
|
||||
|
||||
createSubDir(
|
||||
"hbase.local.dir",
|
||||
testPath, "hbase-local-dir");
|
||||
return testPath;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -169,7 +169,7 @@ public class TestClassLoading {
|
|||
// the classpath is {hbaseSrc}/target/classes.
|
||||
String currentDir = new File(".").getAbsolutePath();
|
||||
String classpath =
|
||||
currentDir + Path.SEPARATOR + "target"+ Path.SEPARATOR + "classes" +
|
||||
currentDir + File.separator + "target"+ File.separator + "classes" +
|
||||
System.getProperty("path.separator") +
|
||||
// Note that the below trick only works if mvn is running the test;
|
||||
// doesn't work in eclipse for example.
|
||||
|
|
@ -304,6 +304,10 @@ public class TestClassLoading {
|
|||
}
|
||||
}
|
||||
|
||||
private String getLocalPath(File file) {
|
||||
return new Path(file.toURI()).toString();
|
||||
}
|
||||
|
||||
@Test
|
||||
// HBASE-3516: Test CP Class loading from local file system
|
||||
public void testClassLoadingFromLocalFS() throws Exception {
|
||||
|
|
@ -312,7 +316,7 @@ public class TestClassLoading {
|
|||
// create a table that references the jar
|
||||
HTableDescriptor htd = new HTableDescriptor(cpName3);
|
||||
htd.addFamily(new HColumnDescriptor("test"));
|
||||
htd.setValue("COPROCESSOR$1", jarFile.toString() + "|" + cpName3 + "|" +
|
||||
htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
|
||||
Coprocessor.PRIORITY_USER);
|
||||
HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
|
||||
admin.createTable(htd);
|
||||
|
|
@ -338,7 +342,7 @@ public class TestClassLoading {
|
|||
// create a table that references the jar
|
||||
HTableDescriptor htd = new HTableDescriptor(cpName4);
|
||||
htd.addFamily(new HColumnDescriptor("test"));
|
||||
htd.setValue("COPROCESSOR$1", jarFile.toString() + "|" + cpName4 + "|" +
|
||||
htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
|
||||
Coprocessor.PRIORITY_USER);
|
||||
HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
|
||||
admin.createTable(htd);
|
||||
|
|
@ -376,9 +380,9 @@ public class TestClassLoading {
|
|||
String cpKey2 = " Coprocessor$2 ";
|
||||
String cpKey3 = " coprocessor$03 ";
|
||||
|
||||
String cpValue1 = jarFile1.toString() + "|" + cpName1 + "|" +
|
||||
String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
|
||||
Coprocessor.PRIORITY_USER;
|
||||
String cpValue2 = jarFile2.toString() + " | " + cpName2 + " | ";
|
||||
String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
|
||||
// load from default class loader
|
||||
String cpValue3 =
|
||||
" | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
|
||||
|
|
@ -393,13 +397,13 @@ public class TestClassLoading {
|
|||
htd.setValue(cpKey3, cpValue3);
|
||||
|
||||
// add 2 coprocessor by using new htd.addCoprocessor() api
|
||||
htd.addCoprocessor(cpName5, new Path(jarFile5.getPath()),
|
||||
htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
|
||||
Coprocessor.PRIORITY_USER, null);
|
||||
Map<String, String> kvs = new HashMap<String, String>();
|
||||
kvs.put("k1", "v1");
|
||||
kvs.put("k2", "v2");
|
||||
kvs.put("k3", "v3");
|
||||
htd.addCoprocessor(cpName6, new Path(jarFile6.getPath()),
|
||||
htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
|
||||
Coprocessor.PRIORITY_USER, kvs);
|
||||
|
||||
HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
|
||||
|
|
|
|||
Loading…
Reference in New Issue