From b33f50e1d40afc64b4f62b38b5f4edbf46a44596 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Sun, 15 Feb 2015 11:21:22 -0800 Subject: [PATCH] HBASE-13044 Configuration option for disabling coprocessor loading Conflicts: hbase-common/src/main/resources/hbase-default.xml hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorConfiguration.java --- .../src/main/resources/hbase-default.xml | 16 ++ .../hbase/coprocessor/CoprocessorHost.java | 11 ++ .../hbase/master/MasterCoprocessorHost.java | 25 ++- .../regionserver/RegionCoprocessorHost.java | 8 + .../RegionServerCoprocessorHost.java | 15 +- .../TestCoprocessorConfiguration.java | 171 ++++++++++++++++++ 6 files changed, 242 insertions(+), 4 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorConfiguration.java diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index f7b0bc3745a..e606994a0db 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -882,6 +882,22 @@ possible configurations would overwhelm and obscure the important. When false (the default), the client will not allow the fallback to SIMPLE authentication, and will abort the connection. + + hbase.coprocessor.enabled + true + Enables or disables coprocessor loading. If 'false' + (disabled), any other coprocessor related configuration will be ignored. + + + + hbase.coprocessor.user.enabled + true + Enables or disables user (aka. table) coprocessor loading. + If 'false' (disabled), any table coprocessor attributes in table + descriptors will be ignored. If "hbase.coprocessor.enabled" is 'false' + this setting has no effect. + + hbase.coprocessor.region.classes diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java index 312a3ae8c05..7f689c9956a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java @@ -74,6 +74,11 @@ public abstract class CoprocessorHost { "hbase.coprocessor.wal.classes"; public static final String ABORT_ON_ERROR_KEY = "hbase.coprocessor.abortonerror"; public static final boolean DEFAULT_ABORT_ON_ERROR = true; + public static final String COPROCESSORS_ENABLED_CONF_KEY = "hbase.coprocessor.enabled"; + public static final boolean DEFAULT_COPROCESSORS_ENABLED = true; + public static final String USER_COPROCESSORS_ENABLED_CONF_KEY = + "hbase.coprocessor.user.enabled"; + public static final boolean DEFAULT_USER_COPROCESSORS_ENABLED = true; private static final Log LOG = LogFactory.getLog(CoprocessorHost.class); protected Abortable abortable; @@ -124,6 +129,12 @@ public abstract class CoprocessorHost { * Called by constructor. */ protected void loadSystemCoprocessors(Configuration conf, String confKey) { + boolean coprocessorsEnabled = conf.getBoolean(COPROCESSORS_ENABLED_CONF_KEY, + DEFAULT_COPROCESSORS_ENABLED); + if (!coprocessorsEnabled) { + return; + } + Class implClass = null; // load default coprocessors from configure file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java index 14c2568d7d0..0fffdab433c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java @@ -19,9 +19,20 @@ package org.apache.hadoop.hbase.master; -import org.apache.hadoop.hbase.classification.InterfaceAudience; +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.Coprocessor; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.coprocessor.*; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; @@ -37,6 +48,8 @@ import java.util.List; public class MasterCoprocessorHost extends CoprocessorHost { + private static final Log LOG = LogFactory.getLog(MasterCoprocessorHost.class); + /** * Coprocessor environment extension providing access to master related * services. @@ -59,10 +72,16 @@ public class MasterCoprocessorHost private MasterServices masterServices; - MasterCoprocessorHost(final MasterServices services, final Configuration conf) { + public MasterCoprocessorHost(final MasterServices services, final Configuration conf) { super(services); this.conf = conf; this.masterServices = services; + // Log the state of coprocessor loading here; should appear only once or + // twice in the daemon log, depending on HBase version, because there is + // only one MasterCoprocessorHost instance in the master process + boolean coprocessorsEnabled = conf.getBoolean(COPROCESSORS_ENABLED_CONF_KEY, + DEFAULT_COPROCESSORS_ENABLED); + LOG.info("System coprocessor loading is " + (coprocessorsEnabled ? "enabled" : "disabled")); loadSystemCoprocessors(conf, MASTER_COPROCESSOR_CONF_KEY); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java index d5235fb9d6d..83aea3e70a3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java @@ -329,6 +329,14 @@ public class RegionCoprocessorHost } void loadTableCoprocessors(final Configuration conf) { + boolean coprocessorsEnabled = conf.getBoolean(COPROCESSORS_ENABLED_CONF_KEY, + DEFAULT_COPROCESSORS_ENABLED); + boolean tableCoprocessorsEnabled = conf.getBoolean(USER_COPROCESSORS_ENABLED_CONF_KEY, + DEFAULT_USER_COPROCESSORS_ENABLED); + if (!(coprocessorsEnabled && tableCoprocessorsEnabled)) { + return; + } + // scan the table attributes for coprocessor load specifications // initialize the coprocessors List configured = new ArrayList(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java index 43a3f32757a..ab8e948b35d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerCoprocessorHost.java @@ -22,6 +22,8 @@ import java.io.IOException; import java.util.Comparator; import java.util.List; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -44,6 +46,8 @@ import org.apache.hadoop.hbase.replication.ReplicationEndpoint; public class RegionServerCoprocessorHost extends CoprocessorHost { + private static final Log LOG = LogFactory.getLog(RegionServerCoprocessorHost.class); + private RegionServerServices rsServices; public RegionServerCoprocessorHost(RegionServerServices rsServices, @@ -51,7 +55,16 @@ public class RegionServerCoprocessorHost extends super(rsServices); this.rsServices = rsServices; this.conf = conf; - // load system default cp's from configuration. + // Log the state of coprocessor loading here; should appear only once or + // twice in the daemon log, depending on HBase version, because there is + // only one RegionServerCoprocessorHost instance in the RS process + boolean coprocessorsEnabled = conf.getBoolean(COPROCESSORS_ENABLED_CONF_KEY, + DEFAULT_COPROCESSORS_ENABLED); + boolean tableCoprocessorsEnabled = conf.getBoolean(USER_COPROCESSORS_ENABLED_CONF_KEY, + DEFAULT_USER_COPROCESSORS_ENABLED); + LOG.info("System coprocessor loading is " + (coprocessorsEnabled ? "enabled" : "disabled")); + LOG.info("Table coprocessor loading is " + + ((coprocessorsEnabled && tableCoprocessorsEnabled) ? "enabled" : "disabled")); loadSystemCoprocessors(conf, REGIONSERVER_COPROCESSOR_CONF_KEY); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorConfiguration.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorConfiguration.java new file mode 100644 index 00000000000..b7a5d4a78c1 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestCoprocessorConfiguration.java @@ -0,0 +1,171 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.coprocessor; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.mockito.Mockito.*; +import static org.junit.Assert.*; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Coprocessor; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.master.MasterCoprocessorHost; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost; +import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost; +import org.apache.hadoop.hbase.regionserver.RegionServerServices; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests for global coprocessor loading configuration + */ +@Category(SmallTests.class) +public class TestCoprocessorConfiguration { + + private static final Configuration CONF = HBaseConfiguration.create(); + static { + CONF.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, + SystemCoprocessor.class.getName()); + CONF.setStrings(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, + SystemCoprocessor.class.getName()); + CONF.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + SystemCoprocessor.class.getName()); + } + private static final TableName TABLENAME = TableName.valueOf("TestCoprocessorConfiguration"); + private static final HRegionInfo REGIONINFO = new HRegionInfo(TABLENAME); + private static final HTableDescriptor TABLEDESC = new HTableDescriptor(TABLENAME); + static { + try { + TABLEDESC.addCoprocessor(TableCoprocessor.class.getName()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + // use atomic types in case coprocessor loading is ever multithreaded, also + // so we can mutate them even though they are declared final here + private static final AtomicBoolean systemCoprocessorLoaded = new AtomicBoolean(); + private static final AtomicBoolean tableCoprocessorLoaded = new AtomicBoolean(); + + public static class SystemCoprocessor implements Coprocessor { + @Override + public void start(CoprocessorEnvironment env) throws IOException { + systemCoprocessorLoaded.set(true); + } + + @Override + public void stop(CoprocessorEnvironment env) throws IOException { } + } + + public static class TableCoprocessor implements Coprocessor { + @Override + public void start(CoprocessorEnvironment env) throws IOException { + tableCoprocessorLoaded.set(true); + } + + @Override + public void stop(CoprocessorEnvironment env) throws IOException { } + } + + @Test + public void testRegionCoprocessorHostDefaults() throws Exception { + Configuration conf = new Configuration(CONF); + HRegion region = mock(HRegion.class); + when(region.getRegionInfo()).thenReturn(REGIONINFO); + when(region.getTableDesc()).thenReturn(TABLEDESC); + RegionServerServices rsServices = mock(RegionServerServices.class); + systemCoprocessorLoaded.set(false); + tableCoprocessorLoaded.set(false); + new RegionCoprocessorHost(region, rsServices, conf); + assertEquals("System coprocessors loading default was not honored", + systemCoprocessorLoaded.get(), + CoprocessorHost.DEFAULT_COPROCESSORS_ENABLED); + assertEquals("Table coprocessors loading default was not honored", + tableCoprocessorLoaded.get(), + CoprocessorHost.DEFAULT_COPROCESSORS_ENABLED && + CoprocessorHost.DEFAULT_USER_COPROCESSORS_ENABLED); + } + + @Test + public void testRegionServerCoprocessorHostDefaults() throws Exception { + Configuration conf = new Configuration(CONF); + RegionServerServices rsServices = mock(RegionServerServices.class); + systemCoprocessorLoaded.set(false); + new RegionServerCoprocessorHost(rsServices, conf); + assertEquals("System coprocessors loading default was not honored", + systemCoprocessorLoaded.get(), + CoprocessorHost.DEFAULT_COPROCESSORS_ENABLED); + } + + @Test + public void testMasterCoprocessorHostDefaults() throws Exception { + Configuration conf = new Configuration(CONF); + MasterServices masterServices = mock(MasterServices.class); + systemCoprocessorLoaded.set(false); + new MasterCoprocessorHost(masterServices, conf); + assertEquals("System coprocessors loading default was not honored", + systemCoprocessorLoaded.get(), + CoprocessorHost.DEFAULT_COPROCESSORS_ENABLED); + } + + @Test + public void testRegionCoprocessorHostAllDisabled() throws Exception { + Configuration conf = new Configuration(CONF); + conf.setBoolean(CoprocessorHost.COPROCESSORS_ENABLED_CONF_KEY, false); + HRegion region = mock(HRegion.class); + when(region.getRegionInfo()).thenReturn(REGIONINFO); + when(region.getTableDesc()).thenReturn(TABLEDESC); + RegionServerServices rsServices = mock(RegionServerServices.class); + systemCoprocessorLoaded.set(false); + tableCoprocessorLoaded.set(false); + new RegionCoprocessorHost(region, rsServices, conf); + assertFalse("System coprocessors should not have been loaded", + systemCoprocessorLoaded.get()); + assertFalse("Table coprocessors should not have been loaded", + tableCoprocessorLoaded.get()); + } + + @Test + public void testRegionCoprocessorHostTableLoadingDisabled() throws Exception { + Configuration conf = new Configuration(CONF); + conf.setBoolean(CoprocessorHost.COPROCESSORS_ENABLED_CONF_KEY, true); // if defaults change + conf.setBoolean(CoprocessorHost.USER_COPROCESSORS_ENABLED_CONF_KEY, false); + HRegion region = mock(HRegion.class); + when(region.getRegionInfo()).thenReturn(REGIONINFO); + when(region.getTableDesc()).thenReturn(TABLEDESC); + RegionServerServices rsServices = mock(RegionServerServices.class); + systemCoprocessorLoaded.set(false); + tableCoprocessorLoaded.set(false); + new RegionCoprocessorHost(region, rsServices, conf); + assertTrue("System coprocessors should have been loaded", + systemCoprocessorLoaded.get()); + assertFalse("Table coprocessors should not have been loaded", + tableCoprocessorLoaded.get()); + } +}