From fd34e19f8e2be03f0fe72a6293b1e5c305f5e55d Mon Sep 17 00:00:00 2001 From: Enis Soztutar Date: Wed, 9 Jan 2013 21:13:06 +0000 Subject: [PATCH] HBASE-6824. Introduce ${hbase.local.dir} and save coprocessor jars there git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1431081 13f79535-47bb-0310-9956-ffa450edef68 --- .../hbase/coprocessor/CoprocessorHost.java | 21 +++++++++++-------- .../hbase/master/MasterCoprocessorHost.java | 1 + .../regionserver/RegionCoprocessorHost.java | 1 + .../src/main/resources/hbase-default.xml | 9 +++++++- .../hadoop/hbase/HBaseTestingUtility.java | 3 +++ .../hbase/coprocessor/TestClassLoading.java | 18 +++++++++------- 6 files changed, 36 insertions(+), 17 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java index 671dfd63cfa..4bb47dea4be 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java @@ -72,6 +72,10 @@ public abstract class CoprocessorHost { public static final String WAL_COPROCESSOR_CONF_KEY = "hbase.coprocessor.wal.classes"; + //coprocessor jars are put under ${hbase.local.dir}/coprocessor/jars/ + private static final String COPROCESSOR_JARS_DIR = File.separator + + "coprocessor" + File.separator + "jars" + File.separator; + private static final Log LOG = LogFactory.getLog(CoprocessorHost.class); /** Ordered set of loaded coprocessors with lock */ protected SortedSet coprocessors = @@ -209,13 +213,13 @@ public abstract class CoprocessorHost { if (!path.toString().endsWith(".jar")) { throw new IOException(path.toString() + ": not a jar file?"); } - FileSystem fs = path.getFileSystem(HBaseConfiguration.create()); - Path dst = new Path(System.getProperty("java.io.tmpdir") + - java.io.File.separator +"." + pathPrefix + + FileSystem fs = path.getFileSystem(this.conf); + File parentDir = new File(this.conf.get("hbase.local.dir") + COPROCESSOR_JARS_DIR); + parentDir.mkdirs(); + File dst = new File(parentDir, "." + pathPrefix + "." + className + "." + System.currentTimeMillis() + ".jar"); - fs.copyToLocalFile(path, dst); - File tmpLocal = new File(dst.toString()); - tmpLocal.deleteOnExit(); + fs.copyToLocalFile(path, new Path(dst.toString())); + dst.deleteOnExit(); // TODO: code weaving goes here @@ -229,7 +233,7 @@ public abstract class CoprocessorHost { // unsurprisingly wants URLs, not URIs; so we will use the deprecated // method which returns URLs for as long as it is available List paths = new ArrayList(); - URL url = new File(dst.toString()).getCanonicalFile().toURL(); + URL url = dst.getCanonicalFile().toURL(); paths.add(url); JarFile jarFile = new JarFile(dst.toString()); @@ -237,8 +241,7 @@ public abstract class CoprocessorHost { while (entries.hasMoreElements()) { JarEntry entry = entries.nextElement(); if (entry.getName().matches("/lib/[^/]+\\.jar")) { - File file = new File(System.getProperty("java.io.tmpdir") + - java.io.File.separator +"." + pathPrefix + + File file = new File(parentDir, "." + pathPrefix + "." + className + "." + System.currentTimeMillis() + "." + entry.getName().substring(5)); IOUtils.copyBytes(jarFile.getInputStream(entry), new FileOutputStream(file), conf, true); file.deleteOnExit(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java index de93998310b..09018bb91ec 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java @@ -63,6 +63,7 @@ public class MasterCoprocessorHost private MasterServices masterServices; MasterCoprocessorHost(final MasterServices services, final Configuration conf) { + this.conf = conf; this.masterServices = services; loadSystemCoprocessors(conf, MASTER_COPROCESSOR_CONF_KEY); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index b0fe580754e..df85151a432 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -134,6 +134,7 @@ public class RegionCoprocessorHost */ public RegionCoprocessorHost(final HRegion region, final RegionServerServices rsServices, final Configuration conf) { + this.conf = conf; this.rsServices = rsServices; this.region = region; this.pathPrefix = Integer.toString(this.region.getRegionInfo().hashCode()); diff --git a/hbase-server/src/main/resources/hbase-default.xml b/hbase-server/src/main/resources/hbase-default.xml index f8d4ae31665..1f690f7a8e4 100644 --- a/hbase-server/src/main/resources/hbase-default.xml +++ b/hbase-server/src/main/resources/hbase-default.xml @@ -49,13 +49,20 @@ hbase.tmp.dir - /tmp/hbase-${user.name} + ${java.io.tmpdir}/hbase-${user.name} Temporary directory on the local filesystem. Change this setting to point to a location more permanent than '/tmp' (The '/tmp' directory is often cleared on machine restart). + + hbase.local.dir + ${hbase.tmp.dir}/local/ + Directory on the local filesystem to be used + as a local storage. + + hbase.master.info.port 60010 diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 0285818e1f2..4ee450ae5dd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -266,6 +266,9 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { "mapred.local.dir", testPath, "mapred-local-dir"); + createSubDir( + "hbase.local.dir", + testPath, "hbase-local-dir"); return testPath; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java index c41ba545248..885ee59ad78 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestClassLoading.java @@ -169,7 +169,7 @@ public class TestClassLoading { // the classpath is {hbaseSrc}/target/classes. String currentDir = new File(".").getAbsolutePath(); String classpath = - currentDir + Path.SEPARATOR + "target"+ Path.SEPARATOR + "classes" + + currentDir + File.separator + "target"+ File.separator + "classes" + System.getProperty("path.separator") + // Note that the below trick only works if mvn is running the test; // doesn't work in eclipse for example. @@ -304,6 +304,10 @@ public class TestClassLoading { } } + private String getLocalPath(File file) { + return new Path(file.toURI()).toString(); + } + @Test // HBASE-3516: Test CP Class loading from local file system public void testClassLoadingFromLocalFS() throws Exception { @@ -312,7 +316,7 @@ public class TestClassLoading { // create a table that references the jar HTableDescriptor htd = new HTableDescriptor(cpName3); htd.addFamily(new HColumnDescriptor("test")); - htd.setValue("COPROCESSOR$1", jarFile.toString() + "|" + cpName3 + "|" + + htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" + Coprocessor.PRIORITY_USER); HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); admin.createTable(htd); @@ -338,7 +342,7 @@ public class TestClassLoading { // create a table that references the jar HTableDescriptor htd = new HTableDescriptor(cpName4); htd.addFamily(new HColumnDescriptor("test")); - htd.setValue("COPROCESSOR$1", jarFile.toString() + "|" + cpName4 + "|" + + htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" + Coprocessor.PRIORITY_USER); HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); admin.createTable(htd); @@ -376,9 +380,9 @@ public class TestClassLoading { String cpKey2 = " Coprocessor$2 "; String cpKey3 = " coprocessor$03 "; - String cpValue1 = jarFile1.toString() + "|" + cpName1 + "|" + + String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" + Coprocessor.PRIORITY_USER; - String cpValue2 = jarFile2.toString() + " | " + cpName2 + " | "; + String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | "; // load from default class loader String cpValue3 = " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v "; @@ -393,13 +397,13 @@ public class TestClassLoading { htd.setValue(cpKey3, cpValue3); // add 2 coprocessor by using new htd.addCoprocessor() api - htd.addCoprocessor(cpName5, new Path(jarFile5.getPath()), + htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)), Coprocessor.PRIORITY_USER, null); Map kvs = new HashMap(); kvs.put("k1", "v1"); kvs.put("k2", "v2"); kvs.put("k3", "v3"); - htd.addCoprocessor(cpName6, new Path(jarFile6.getPath()), + htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)), Coprocessor.PRIORITY_USER, kvs); HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();