From 124ac4d6c4a19881170d01a374502057ac1a22a6 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Tue, 10 Sep 2019 11:14:23 +0800 Subject: [PATCH] Revert "HBASE-22676 Move all the code in hbase-rsgroup to hbase-server and remove hbase-rsgroup module (#399)" This reverts commit f7bbdde9756e67f23a11efd0a6dac9dc58843058. --- .../src/main/assembly/components.xml | 8 + .../src/main/assembly/hadoop-two-compat.xml | 1 + hbase-it/pom.xml | 10 + hbase-rsgroup/README.txt | 13 + hbase-rsgroup/pom.xml | 256 ++++++++ .../hadoop/hbase/rsgroup/RSGroupAdmin.java | 0 .../hbase/rsgroup/RSGroupAdminClient.java | 0 .../hbase/rsgroup/RSGroupAdminEndpoint.java | 559 ++++++++++++++++++ .../hbase/rsgroup/RSGroupAdminServer.java | 0 .../rsgroup/RSGroupBasedLoadBalancer.java | 24 +- .../rsgroup/RSGroupMajorCompactionTTL.java | 0 .../hbase/rsgroup/RSGroupableBalancer.java | 0 .../balancer/RSGroupableBalancerTestBase.java | 0 .../TestRSGroupBasedLoadBalancer.java | 0 ...rWithStochasticLoadBalancerAsInternal.java | 0 .../hbase/rsgroup/TestEnableRSGroups.java | 0 .../TestRSGroupMajorCompactionTTL.java | 0 .../hbase/rsgroup/TestRSGroupsAdmin1.java | 0 .../hbase/rsgroup/TestRSGroupsAdmin2.java | 0 .../hbase/rsgroup/TestRSGroupsBalance.java | 0 .../hbase/rsgroup/TestRSGroupsBase.java | 0 .../hbase/rsgroup/TestRSGroupsBasics.java | 0 .../hbase/rsgroup/TestRSGroupsKillRS.java | 0 .../rsgroup/TestRSGroupsOfflineMode.java | 0 .../hbase/rsgroup/TestRSGroupsWithACL.java | 77 ++- .../rsgroup/VerifyingRSGroupAdminClient.java | 0 .../src/test/resources/hbase-site.xml | 32 + .../src/test/resources/log4j.properties | 68 +++ .../hbase/rsgroup/RSGroupAdminEndpoint.java | 226 ------- .../rsgroup/RSGroupAdminServiceImpl.java | 378 ------------ hbase-shell/pom.xml | 35 ++ pom.xml | 24 + 32 files changed, 1064 insertions(+), 647 deletions(-) create mode 100644 hbase-rsgroup/README.txt create mode 100644 hbase-rsgroup/pom.xml rename {hbase-server => hbase-rsgroup}/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java (100%) rename {hbase-server => hbase-rsgroup}/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java (100%) create mode 100644 hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java rename {hbase-server => hbase-rsgroup}/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java (100%) rename {hbase-server => hbase-rsgroup}/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java (98%) rename {hbase-server => hbase-rsgroup}/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupMajorCompactionTTL.java (100%) rename {hbase-server => hbase-rsgroup}/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupableBalancer.java (100%) rename {hbase-server => hbase-rsgroup}/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java (100%) rename {hbase-server => hbase-rsgroup}/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java (100%) rename {hbase-server => hbase-rsgroup}/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java (100%) rename {hbase-server => hbase-rsgroup}/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroups.java (100%) rename {hbase-server => hbase-rsgroup}/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java (100%) rename {hbase-server => hbase-rsgroup}/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java (100%) rename {hbase-server => hbase-rsgroup}/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java (100%) rename {hbase-server => hbase-rsgroup}/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java (100%) rename {hbase-server => hbase-rsgroup}/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java (100%) rename {hbase-server => hbase-rsgroup}/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java (100%) rename {hbase-server => hbase-rsgroup}/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java (100%) rename {hbase-server => hbase-rsgroup}/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java (100%) rename {hbase-server => hbase-rsgroup}/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java (81%) rename {hbase-server => hbase-rsgroup}/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java (100%) create mode 100644 hbase-rsgroup/src/test/resources/hbase-site.xml create mode 100644 hbase-rsgroup/src/test/resources/log4j.properties delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServiceImpl.java diff --git a/hbase-assembly/src/main/assembly/components.xml b/hbase-assembly/src/main/assembly/components.xml index 4dd85ef0170..2eb16e76497 100644 --- a/hbase-assembly/src/main/assembly/components.xml +++ b/hbase-assembly/src/main/assembly/components.xml @@ -144,6 +144,14 @@ 0644 + + ${project.basedir}/../hbase-rsgroup/target/ + lib + + ${rsgroup.test.jar} + + 0644 + ${project.basedir}/../hbase-mapreduce/target/ lib diff --git a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml index 6a21b920873..91d374987a6 100644 --- a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml +++ b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml @@ -52,6 +52,7 @@ org.apache.hbase:hbase-protocol-shaded org.apache.hbase:hbase-replication org.apache.hbase:hbase-rest + org.apache.hbase:hbase-rsgroup org.apache.hbase:hbase-server org.apache.hbase:hbase-shell org.apache.hbase:hbase-testing-util diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml index 6b173f0ffd6..0fe578a99f9 100644 --- a/hbase-it/pom.xml +++ b/hbase-it/pom.xml @@ -181,6 +181,16 @@ test-jar test + + org.apache.hbase + hbase-rsgroup + + + org.apache.hbase + hbase-rsgroup + test-jar + test + org.apache.hbase hbase-server diff --git a/hbase-rsgroup/README.txt b/hbase-rsgroup/README.txt new file mode 100644 index 00000000000..b24aee65060 --- /dev/null +++ b/hbase-rsgroup/README.txt @@ -0,0 +1,13 @@ +ON PROTOBUFS +This maven module has protobuf definition files ('.protos') used by hbase +Coprocessor Endpoints that ship with hbase core including tests. Coprocessor +Endpoints are meant to be standalone, independent code not reliant on hbase +internals. They define their Service using protobuf. The protobuf version +they use can be distinct from that used by HBase internally since HBase started +shading its protobuf references. Endpoints have no access to the shaded protobuf +hbase uses. They do have access to the content of hbase-protocol -- the +.protos found in here -- but avoid using as much of this as you can as it is +liable to change. + +Generation of java files from protobuf .proto files included here is done as +part of the build. diff --git a/hbase-rsgroup/pom.xml b/hbase-rsgroup/pom.xml new file mode 100644 index 00000000000..7b52af7dddb --- /dev/null +++ b/hbase-rsgroup/pom.xml @@ -0,0 +1,256 @@ + + + + 4.0.0 + + hbase-build-configuration + org.apache.hbase + 3.0.0-SNAPSHOT + ../hbase-build-configuration + + hbase-rsgroup + Apache HBase - RSGroup + Regionserver Groups for HBase + + + + + maven-assembly-plugin + + true + + + + + org.apache.maven.plugins + maven-source-plugin + + + org.apache.maven.plugins + maven-checkstyle-plugin + + true + + + + net.revelc.code + warbucks-maven-plugin + + + + + + + org.apache.hbase + hbase-annotations + test-jar + test + + + org.apache.hbase + hbase-client + + + org.apache.hbase + hbase-server + + + org.apache.hbase + hbase-common + + + org.apache.hbase + hbase-procedure + + + org.apache.hbase + hbase-protocol + + + org.apache.hbase + hbase-protocol-shaded + + + org.apache.hbase + hbase-testing-util + test + + + + org.apache.commons + commons-lang3 + + + org.slf4j + slf4j-api + + + org.apache.hbase.thirdparty + hbase-shaded-miscellaneous + + + org.apache.zookeeper + zookeeper + + + log4j + log4j + test + + + org.mockito + mockito-core + test + + + junit + junit + test + + + + + + skipRSGroupTests + + + skipRSGroupTests + + + + true + true + + + + + hadoop-2.0 + + + + + !hadoop.profile + + + + + com.github.stephenc.findbugs + findbugs-annotations + true + + + org.apache.hadoop + hadoop-common + + + net.java.dev.jets3t + jets3t + + + javax.servlet.jsp + jsp-api + + + org.mortbay.jetty + jetty + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-json + + + javax.servlet + servlet-api + + + tomcat + jasper-compiler + + + tomcat + jasper-runtime + + + com.google.code.findbugs + jsr305 + + + + + + + + hadoop-3.0 + + + hadoop.profile + 3.0 + + + + 3.0-SNAPSHOT + + + + org.apache.hadoop + hadoop-common + + + + + eclipse-specific + + + m2e.version + + + + + + + + org.eclipse.m2e + lifecycle-mapping + + + + + + + + + + + + + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java rename to hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java rename to hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java new file mode 100644 index 00000000000..8c6e0100b41 --- /dev/null +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java @@ -0,0 +1,559 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rsgroup; + +import com.google.protobuf.RpcCallback; +import com.google.protobuf.RpcController; +import com.google.protobuf.Service; +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MasterNotRunningException; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.PleaseHoldException; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.SnapshotDescription; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.constraint.ConstraintException; +import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor; +import org.apache.hadoop.hbase.coprocessor.HasMasterServices; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.MasterObserver; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; +import org.apache.hadoop.hbase.ipc.RpcServer; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.net.Address; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.AddRSGroupRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.AddRSGroupResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfTableRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfTableResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersAndTablesRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersAndTablesResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveTablesRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveTablesResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RSGroupAdminService; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveServersRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveServersResponse; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.hbase.security.access.AccessChecker; +import org.apache.hadoop.hbase.security.access.Permission.Action; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.Sets; + +// TODO: Encapsulate MasterObserver functions into separate subclass. +@CoreCoprocessor +@InterfaceAudience.Private +public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver { + private static final Logger LOG = LoggerFactory.getLogger(RSGroupAdminEndpoint.class); + + private MasterServices master = null; + // Only instance of RSGroupInfoManager. RSGroup aware load balancers ask for this instance on + // their setup. + private RSGroupInfoManager groupInfoManager; + private RSGroupAdminServer groupAdminServer; + private final RSGroupAdminService groupAdminService = new RSGroupAdminServiceImpl(); + private AccessChecker accessChecker; + + /** Provider for mapping principal names to Users */ + private UserProvider userProvider; + + @Override + public void start(CoprocessorEnvironment env) throws IOException { + if (!(env instanceof HasMasterServices)) { + throw new IOException("Does not implement HMasterServices"); + } + + master = ((HasMasterServices)env).getMasterServices(); + groupInfoManager = RSGroupInfoManagerImpl.getInstance(master); + groupAdminServer = new RSGroupAdminServer(master, groupInfoManager); + Class clazz = + master.getConfiguration().getClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, null); + if (!RSGroupableBalancer.class.isAssignableFrom(clazz)) { + throw new IOException("Configured balancer does not support RegionServer groups."); + } + accessChecker = ((HasMasterServices) env).getMasterServices().getAccessChecker(); + + // set the user-provider. + this.userProvider = UserProvider.instantiate(env.getConfiguration()); + } + + @Override + public void stop(CoprocessorEnvironment env) { + } + + @Override + public Iterable getServices() { + return Collections.singleton(groupAdminService); + } + + @Override + public Optional getMasterObserver() { + return Optional.of(this); + } + + RSGroupInfoManager getGroupInfoManager() { + return groupInfoManager; + } + + /** + * Implementation of RSGroupAdminService defined in RSGroupAdmin.proto. + * This class calls {@link RSGroupAdminServer} for actual work, converts result to protocol + * buffer response, handles exceptions if any occurred and then calls the {@code RpcCallback} with + * the response. + */ + private class RSGroupAdminServiceImpl extends RSGroupAdminProtos.RSGroupAdminService { + @Override + public void getRSGroupInfo(RpcController controller, + GetRSGroupInfoRequest request, RpcCallback done) { + GetRSGroupInfoResponse.Builder builder = GetRSGroupInfoResponse.newBuilder(); + String groupName = request.getRSGroupName(); + LOG.info(master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, group=" + + groupName); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preGetRSGroupInfo(groupName); + } + checkPermission("getRSGroupInfo"); + RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); + if (rsGroupInfo != null) { + builder.setRSGroupInfo(ProtobufUtil.toProtoGroupInfo(rsGroupInfo)); + } + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postGetRSGroupInfo(groupName); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void getRSGroupInfoOfTable(RpcController controller, + GetRSGroupInfoOfTableRequest request, RpcCallback done) { + GetRSGroupInfoOfTableResponse.Builder builder = GetRSGroupInfoOfTableResponse.newBuilder(); + TableName tableName = ProtobufUtil.toTableName(request.getTableName()); + LOG.info(master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, table=" + + tableName); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preGetRSGroupInfoOfTable(tableName); + } + checkPermission("getRSGroupInfoOfTable"); + RSGroupInfo RSGroupInfo = groupAdminServer.getRSGroupInfoOfTable(tableName); + if (RSGroupInfo != null) { + builder.setRSGroupInfo(ProtobufUtil.toProtoGroupInfo(RSGroupInfo)); + } + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postGetRSGroupInfoOfTable(tableName); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void moveServers(RpcController controller, MoveServersRequest request, + RpcCallback done) { + MoveServersResponse.Builder builder = MoveServersResponse.newBuilder(); + Set
hostPorts = Sets.newHashSet(); + for (HBaseProtos.ServerName el : request.getServersList()) { + hostPorts.add(Address.fromParts(el.getHostName(), el.getPort())); + } + LOG.info(master.getClientIdAuditPrefix() + " move servers " + hostPorts +" to rsgroup " + + request.getTargetGroup()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preMoveServers(hostPorts, request.getTargetGroup()); + } + checkPermission("moveServers"); + groupAdminServer.moveServers(hostPorts, request.getTargetGroup()); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postMoveServers(hostPorts, request.getTargetGroup()); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void moveTables(RpcController controller, MoveTablesRequest request, + RpcCallback done) { + MoveTablesResponse.Builder builder = MoveTablesResponse.newBuilder(); + Set tables = new HashSet<>(request.getTableNameList().size()); + for (HBaseProtos.TableName tableName : request.getTableNameList()) { + tables.add(ProtobufUtil.toTableName(tableName)); + } + LOG.info(master.getClientIdAuditPrefix() + " move tables " + tables +" to rsgroup " + + request.getTargetGroup()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preMoveTables(tables, request.getTargetGroup()); + } + checkPermission("moveTables"); + groupAdminServer.moveTables(tables, request.getTargetGroup()); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postMoveTables(tables, request.getTargetGroup()); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void addRSGroup(RpcController controller, AddRSGroupRequest request, + RpcCallback done) { + AddRSGroupResponse.Builder builder = AddRSGroupResponse.newBuilder(); + LOG.info(master.getClientIdAuditPrefix() + " add rsgroup " + request.getRSGroupName()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preAddRSGroup(request.getRSGroupName()); + } + checkPermission("addRSGroup"); + groupAdminServer.addRSGroup(request.getRSGroupName()); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postAddRSGroup(request.getRSGroupName()); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void removeRSGroup(RpcController controller, + RemoveRSGroupRequest request, RpcCallback done) { + RemoveRSGroupResponse.Builder builder = + RemoveRSGroupResponse.newBuilder(); + LOG.info(master.getClientIdAuditPrefix() + " remove rsgroup " + request.getRSGroupName()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preRemoveRSGroup(request.getRSGroupName()); + } + checkPermission("removeRSGroup"); + groupAdminServer.removeRSGroup(request.getRSGroupName()); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postRemoveRSGroup(request.getRSGroupName()); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void balanceRSGroup(RpcController controller, + BalanceRSGroupRequest request, RpcCallback done) { + BalanceRSGroupResponse.Builder builder = BalanceRSGroupResponse.newBuilder(); + LOG.info(master.getClientIdAuditPrefix() + " balance rsgroup, group=" + + request.getRSGroupName()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preBalanceRSGroup(request.getRSGroupName()); + } + checkPermission("balanceRSGroup"); + boolean balancerRan = groupAdminServer.balanceRSGroup(request.getRSGroupName()); + builder.setBalanceRan(balancerRan); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postBalanceRSGroup(request.getRSGroupName(), + balancerRan); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + builder.setBalanceRan(false); + } + done.run(builder.build()); + } + + @Override + public void listRSGroupInfos(RpcController controller, + ListRSGroupInfosRequest request, RpcCallback done) { + ListRSGroupInfosResponse.Builder builder = ListRSGroupInfosResponse.newBuilder(); + LOG.info(master.getClientIdAuditPrefix() + " list rsgroup"); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preListRSGroups(); + } + checkPermission("listRSGroup"); + for (RSGroupInfo RSGroupInfo : groupAdminServer.listRSGroups()) { + builder.addRSGroupInfo(ProtobufUtil.toProtoGroupInfo(RSGroupInfo)); + } + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postListRSGroups(); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void getRSGroupInfoOfServer(RpcController controller, + GetRSGroupInfoOfServerRequest request, RpcCallback done) { + GetRSGroupInfoOfServerResponse.Builder builder = GetRSGroupInfoOfServerResponse.newBuilder(); + Address hp = Address.fromParts(request.getServer().getHostName(), + request.getServer().getPort()); + LOG.info(master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, server=" + + hp); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preGetRSGroupInfoOfServer(hp); + } + checkPermission("getRSGroupInfoOfServer"); + RSGroupInfo info = groupAdminServer.getRSGroupOfServer(hp); + if (info != null) { + builder.setRSGroupInfo(ProtobufUtil.toProtoGroupInfo(info)); + } + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postGetRSGroupInfoOfServer(hp); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void moveServersAndTables(RpcController controller, + MoveServersAndTablesRequest request, RpcCallback done) { + MoveServersAndTablesResponse.Builder builder = MoveServersAndTablesResponse.newBuilder(); + Set
hostPorts = Sets.newHashSet(); + for (HBaseProtos.ServerName el : request.getServersList()) { + hostPorts.add(Address.fromParts(el.getHostName(), el.getPort())); + } + Set tables = new HashSet<>(request.getTableNameList().size()); + for (HBaseProtos.TableName tableName : request.getTableNameList()) { + tables.add(ProtobufUtil.toTableName(tableName)); + } + LOG.info(master.getClientIdAuditPrefix() + " move servers " + hostPorts + + " and tables " + tables + " to rsgroup" + request.getTargetGroup()); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preMoveServersAndTables(hostPorts, tables, + request.getTargetGroup()); + } + checkPermission("moveServersAndTables"); + groupAdminServer.moveServersAndTables(hostPorts, tables, request.getTargetGroup()); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postMoveServersAndTables(hostPorts, tables, + request.getTargetGroup()); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + + @Override + public void removeServers(RpcController controller, + RemoveServersRequest request, + RpcCallback done) { + RemoveServersResponse.Builder builder = + RemoveServersResponse.newBuilder(); + Set
servers = Sets.newHashSet(); + for (HBaseProtos.ServerName el : request.getServersList()) { + servers.add(Address.fromParts(el.getHostName(), el.getPort())); + } + LOG.info(master.getClientIdAuditPrefix() + + " remove decommissioned servers from rsgroup: " + servers); + try { + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().preRemoveServers(servers); + } + checkPermission("removeServers"); + groupAdminServer.removeServers(servers); + if (master.getMasterCoprocessorHost() != null) { + master.getMasterCoprocessorHost().postRemoveServers(servers); + } + } catch (IOException e) { + CoprocessorRpcUtils.setControllerException(controller, e); + } + done.run(builder.build()); + } + } + + boolean rsgroupHasServersOnline(TableDescriptor desc) throws IOException { + String groupName; + try { + groupName = + master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString()) + .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); + if (groupName == null) { + groupName = RSGroupInfo.DEFAULT_GROUP; + } + } catch (MasterNotRunningException | PleaseHoldException e) { + LOG.info("Master has not initialized yet; temporarily using default RSGroup '" + + RSGroupInfo.DEFAULT_GROUP + "' for deploy of system table"); + groupName = RSGroupInfo.DEFAULT_GROUP; + } + + RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); + if (rsGroupInfo == null) { + throw new ConstraintException( + "Default RSGroup (" + groupName + ") for this table's " + "namespace does not exist."); + } + + for (ServerName onlineServer : master.getServerManager().createDestinationServersList()) { + if (rsGroupInfo.getServers().contains(onlineServer.getAddress())) { + return true; + } + } + return false; + } + + void assignTableToGroup(TableDescriptor desc) throws IOException { + String groupName = + master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString()) + .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); + if (groupName == null) { + groupName = RSGroupInfo.DEFAULT_GROUP; + } + RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); + if (rsGroupInfo == null) { + throw new ConstraintException("Default RSGroup (" + groupName + ") for this table's " + + "namespace does not exist."); + } + if (!rsGroupInfo.containsTable(desc.getTableName())) { + LOG.debug("Pre-moving table " + desc.getTableName() + " to RSGroup " + groupName); + groupAdminServer.moveTables(Sets.newHashSet(desc.getTableName()), groupName); + } + } + + ///////////////////////////////////////////////////////////////////////////// + // MasterObserver overrides + ///////////////////////////////////////////////////////////////////////////// + + @Override + public void preCreateTableAction( + final ObserverContext ctx, + final TableDescriptor desc, + final RegionInfo[] regions) throws IOException { + if (!desc.getTableName().isSystemTable() && !rsgroupHasServersOnline(desc)) { + throw new HBaseIOException("No online servers in the rsgroup, which table " + + desc.getTableName().getNameAsString() + " belongs to"); + } + } + + // Assign table to default RSGroup. + @Override + public void postCreateTable(ObserverContext ctx, + TableDescriptor desc, RegionInfo[] regions) throws IOException { + assignTableToGroup(desc); + } + + // Remove table from its RSGroup. + @Override + public void postDeleteTable(ObserverContext ctx, + TableName tableName) throws IOException { + try { + RSGroupInfo group = groupAdminServer.getRSGroupInfoOfTable(tableName); + if (group != null) { + LOG.debug(String.format("Removing deleted table '%s' from rsgroup '%s'", tableName, + group.getName())); + groupAdminServer.moveTables(Sets.newHashSet(tableName), null); + } + } catch (IOException ex) { + LOG.debug("Failed to perform RSGroup information cleanup for table: " + tableName, ex); + } + } + + @Override + public void preCreateNamespace(ObserverContext ctx, + NamespaceDescriptor ns) throws IOException { + String group = ns.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); + if(group != null && groupAdminServer.getRSGroupInfo(group) == null) { + throw new ConstraintException("Region server group "+group+" does not exit"); + } + } + + @Override + public void preModifyNamespace(ObserverContext ctx, + NamespaceDescriptor currentNsDesc, NamespaceDescriptor newNsDesc) throws IOException { + preCreateNamespace(ctx, newNsDesc); + } + + @Override + public void preCloneSnapshot(ObserverContext ctx, + SnapshotDescription snapshot, TableDescriptor desc) throws IOException { + assignTableToGroup(desc); + } + + @Override + public void postClearDeadServers(ObserverContext ctx, + List servers, List notClearedServers) + throws IOException { + Set
clearedServer = servers.stream(). + filter(server -> !notClearedServers.contains(server)). + map(ServerName::getAddress). + collect(Collectors.toSet()); + if(!clearedServer.isEmpty()) { + groupAdminServer.removeServers(clearedServer); + } + } + + public void checkPermission(String request) throws IOException { + accessChecker.requirePermission(getActiveUser(), request, null, Action.ADMIN); + } + + /** + * Returns the active user to which authorization checks should be applied. + * If we are in the context of an RPC call, the remote user is used, + * otherwise the currently logged in user is used. + */ + private User getActiveUser() throws IOException { + // for non-rpc handling, fallback to system user + Optional optionalUser = RpcServer.getRequestUser(); + if (optionalUser.isPresent()) { + return optionalUser.get(); + } + return userProvider.getCurrent(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java rename to hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java similarity index 98% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java rename to hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java index f585a851b4c..76c96f55402 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.hadoop.hbase.rsgroup; import java.io.IOException; @@ -55,17 +56,18 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.com.google.common.collect.Maps; /** - * GroupBasedLoadBalancer, used when Region Server Grouping is configured (HBase-6721) It does - * region balance based on a table's group membership. - *

- * Most assignment methods contain two exclusive code paths: Online - when the group table is online - * and Offline - when it is unavailable. - *

- * During Offline, assignments are assigned based on cached information in zookeeper. If unavailable - * (ie bootstrap) then regions are assigned randomly. - *

- * Once the GROUP table has been assigned, the balancer switches to Online and will then start - * providing appropriate assignments for user tables. + * GroupBasedLoadBalancer, used when Region Server Grouping is configured (HBase-6721) + * It does region balance based on a table's group membership. + * + * Most assignment methods contain two exclusive code paths: Online - when the group + * table is online and Offline - when it is unavailable. + * + * During Offline, assignments are assigned based on cached information in zookeeper. + * If unavailable (ie bootstrap) then regions are assigned randomly. + * + * Once the GROUP table has been assigned, the balancer switches to Online and will then + * start providing appropriate assignments for user tables. + * */ @InterfaceAudience.Private public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupMajorCompactionTTL.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupMajorCompactionTTL.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupMajorCompactionTTL.java rename to hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupMajorCompactionTTL.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupableBalancer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupableBalancer.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupableBalancer.java rename to hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupableBalancer.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java rename to hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java rename to hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java rename to hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroups.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroups.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroups.java rename to hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroups.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java rename to hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java rename to hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java rename to hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java rename to hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java rename to hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java rename to hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java rename to hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java rename to hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java similarity index 81% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java rename to hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java index c1c157a303e..0278e3cfbfa 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.security.access.SecureTestUtil; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.SecurityTests; import org.apache.hadoop.hbase.util.Bytes; + import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -48,11 +49,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Performs authorization checks for rsgroup operations, according to different levels of authorized - * users. + * Performs authorization checks for rsgroup operations, according to different + * levels of authorized users. */ -@Category({ SecurityTests.class, MediumTests.class }) -public class TestRSGroupsWithACL extends SecureTestUtil { +@Category({SecurityTests.class, MediumTests.class}) +public class TestRSGroupsWithACL extends SecureTestUtil{ @ClassRule public static final HBaseClassTestRule CLASS_RULE = @@ -97,7 +98,8 @@ public class TestRSGroupsWithACL extends SecureTestUtil { public static void setupBeforeClass() throws Exception { // setup configuration conf = TEST_UTIL.getConfiguration(); - conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, RSGroupBasedLoadBalancer.class.getName()); + conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + RSGroupBasedLoadBalancer.class.getName()); // Enable security enableSecurity(conf); // Verify enableSecurity sets up what we require @@ -106,8 +108,8 @@ public class TestRSGroupsWithACL extends SecureTestUtil { configureRSGroupAdminEndpoint(conf); TEST_UTIL.startMiniCluster(); - rsGroupAdminEndpoint = (RSGroupAdminEndpoint) TEST_UTIL.getMiniHBaseCluster().getMaster() - .getMasterCoprocessorHost().findCoprocessor(RSGroupAdminEndpoint.class.getName()); + rsGroupAdminEndpoint = (RSGroupAdminEndpoint) TEST_UTIL.getMiniHBaseCluster().getMaster(). + getMasterCoprocessorHost().findCoprocessor(RSGroupAdminEndpoint.class.getName()); // Wait for the ACL table to become available TEST_UTIL.waitUntilAllRegionsAssigned(PermissionStorage.ACL_TABLE_NAME); @@ -139,21 +141,31 @@ public class TestRSGroupsWithACL extends SecureTestUtil { cfd.setMaxVersions(100); tableBuilder.setColumnFamily(cfd.build()); tableBuilder.setValue(TableDescriptorBuilder.OWNER, USER_OWNER.getShortName()); - createTable(TEST_UTIL, tableBuilder.build(), new byte[][] { Bytes.toBytes("s") }); + createTable(TEST_UTIL, tableBuilder.build(), + new byte[][] { Bytes.toBytes("s") }); // Set up initial grants - grantGlobal(TEST_UTIL, USER_ADMIN.getShortName(), Permission.Action.ADMIN, - Permission.Action.CREATE, Permission.Action.READ, Permission.Action.WRITE); + grantGlobal(TEST_UTIL, USER_ADMIN.getShortName(), + Permission.Action.ADMIN, + Permission.Action.CREATE, + Permission.Action.READ, + Permission.Action.WRITE); - grantOnTable(TEST_UTIL, USER_RW.getShortName(), TEST_TABLE, TEST_FAMILY, null, - Permission.Action.READ, Permission.Action.WRITE); + grantOnTable(TEST_UTIL, USER_RW.getShortName(), + TEST_TABLE, TEST_FAMILY, null, + Permission.Action.READ, + Permission.Action.WRITE); // USER_CREATE is USER_RW plus CREATE permissions - grantOnTable(TEST_UTIL, USER_CREATE.getShortName(), TEST_TABLE, null, null, - Permission.Action.CREATE, Permission.Action.READ, Permission.Action.WRITE); + grantOnTable(TEST_UTIL, USER_CREATE.getShortName(), + TEST_TABLE, null, null, + Permission.Action.CREATE, + Permission.Action.READ, + Permission.Action.WRITE); - grantOnTable(TEST_UTIL, USER_RO.getShortName(), TEST_TABLE, TEST_FAMILY, null, - Permission.Action.READ); + grantOnTable(TEST_UTIL, USER_RO.getShortName(), + TEST_TABLE, TEST_FAMILY, null, + Permission.Action.READ); grantGlobal(TEST_UTIL, toGroupEntry(GROUP_ADMIN), Permission.Action.ADMIN); grantGlobal(TEST_UTIL, toGroupEntry(GROUP_CREATE), Permission.Action.CREATE); @@ -162,8 +174,8 @@ public class TestRSGroupsWithACL extends SecureTestUtil { assertEquals(4, PermissionStorage.getTablePermissions(conf, TEST_TABLE).size()); try { - assertEquals(4, - AccessControlClient.getUserPermissions(systemUserConnection, TEST_TABLE.toString()).size()); + assertEquals(4, AccessControlClient.getUserPermissions(systemUserConnection, + TEST_TABLE.toString()).size()); } catch (AssertionError e) { fail(e.getMessage()); } catch (Throwable e) { @@ -198,13 +210,14 @@ public class TestRSGroupsWithACL extends SecureTestUtil { coprocessors += "," + currentCoprocessors; } conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, coprocessors); - conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, RSGroupBasedLoadBalancer.class.getName()); + conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + RSGroupBasedLoadBalancer.class.getName()); } @Test public void testGetRSGroupInfo() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.getGroupAdminService().checkPermission("getRSGroupInfo"); + rsGroupAdminEndpoint.checkPermission("getRSGroupInfo"); return null; }; @@ -214,7 +227,7 @@ public class TestRSGroupsWithACL extends SecureTestUtil { @Test public void testGetRSGroupInfoOfTable() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.getGroupAdminService().checkPermission("getRSGroupInfoOfTable"); + rsGroupAdminEndpoint.checkPermission("getRSGroupInfoOfTable"); return null; }; @@ -224,7 +237,7 @@ public class TestRSGroupsWithACL extends SecureTestUtil { @Test public void testMoveServers() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.getGroupAdminService().checkPermission("moveServers"); + rsGroupAdminEndpoint.checkPermission("moveServers"); return null; }; @@ -234,7 +247,7 @@ public class TestRSGroupsWithACL extends SecureTestUtil { @Test public void testMoveTables() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.getGroupAdminService().checkPermission("moveTables"); + rsGroupAdminEndpoint.checkPermission("moveTables"); return null; }; @@ -244,7 +257,7 @@ public class TestRSGroupsWithACL extends SecureTestUtil { @Test public void testAddRSGroup() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.getGroupAdminService().checkPermission("addRSGroup"); + rsGroupAdminEndpoint.checkPermission("addRSGroup"); return null; }; @@ -254,7 +267,7 @@ public class TestRSGroupsWithACL extends SecureTestUtil { @Test public void testRemoveRSGroup() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.getGroupAdminService().checkPermission("removeRSGroup"); + rsGroupAdminEndpoint.checkPermission("removeRSGroup"); return null; }; @@ -264,7 +277,7 @@ public class TestRSGroupsWithACL extends SecureTestUtil { @Test public void testBalanceRSGroup() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.getGroupAdminService().checkPermission("balanceRSGroup"); + rsGroupAdminEndpoint.checkPermission("balanceRSGroup"); return null; }; @@ -274,7 +287,7 @@ public class TestRSGroupsWithACL extends SecureTestUtil { @Test public void testListRSGroup() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.getGroupAdminService().checkPermission("listRSGroup"); + rsGroupAdminEndpoint.checkPermission("listRSGroup"); return null; }; @@ -284,7 +297,7 @@ public class TestRSGroupsWithACL extends SecureTestUtil { @Test public void testGetRSGroupInfoOfServer() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.getGroupAdminService().checkPermission("getRSGroupInfoOfServer"); + rsGroupAdminEndpoint.checkPermission("getRSGroupInfoOfServer"); return null; }; @@ -294,7 +307,7 @@ public class TestRSGroupsWithACL extends SecureTestUtil { @Test public void testMoveServersAndTables() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.getGroupAdminService().checkPermission("moveServersAndTables"); + rsGroupAdminEndpoint.checkPermission("moveServersAndTables"); return null; }; @@ -304,7 +317,7 @@ public class TestRSGroupsWithACL extends SecureTestUtil { @Test public void testRemoveServers() throws Exception { AccessTestAction action = () -> { - rsGroupAdminEndpoint.getGroupAdminService().checkPermission("removeServers"); + rsGroupAdminEndpoint.checkPermission("removeServers"); return null; }; @@ -313,7 +326,7 @@ public class TestRSGroupsWithACL extends SecureTestUtil { private void validateAdminPermissions(AccessTestAction action) throws Exception { verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); - verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, - USER_GROUP_WRITE, USER_GROUP_CREATE); + verifyDenied(action, USER_CREATE, USER_OWNER, USER_RW, USER_RO, + USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java rename to hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java diff --git a/hbase-rsgroup/src/test/resources/hbase-site.xml b/hbase-rsgroup/src/test/resources/hbase-site.xml new file mode 100644 index 00000000000..99d2ab8d1fb --- /dev/null +++ b/hbase-rsgroup/src/test/resources/hbase-site.xml @@ -0,0 +1,32 @@ + + + + + + hbase.defaults.for.version.skip + true + + + hbase.hconnection.threads.keepalivetime + 3 + + diff --git a/hbase-rsgroup/src/test/resources/log4j.properties b/hbase-rsgroup/src/test/resources/log4j.properties new file mode 100644 index 00000000000..c322699ced2 --- /dev/null +++ b/hbase-rsgroup/src/test/resources/log4j.properties @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties +hbase.root.logger=INFO,console +hbase.log.dir=. +hbase.log.file=hbase.log + +# Define the root logger to the system property "hbase.root.logger". +log4j.rootLogger=${hbase.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout +# Debugging Pattern format +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n + + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n + +# Custom Logging levels + +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG + +log4j.logger.org.apache.hadoop=WARN +log4j.logger.org.apache.zookeeper=ERROR +log4j.logger.org.apache.hadoop.hbase=DEBUG + +#These settings are workarounds against spurious logs from the minicluster. +#See HBASE-4709 +log4j.logger.org.apache.hadoop.metrics2.impl.MetricsConfig=WARN +log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSinkAdapter=WARN +log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=WARN +log4j.logger.org.apache.hadoop.metrics2.util.MBeans=WARN +# Enable this to get detailed connection error/retry logging. +# log4j.logger.org.apache.hadoop.hbase.client.ConnectionImplementation=TRACE diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java deleted file mode 100644 index 2d5af04c389..00000000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java +++ /dev/null @@ -1,226 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.rsgroup; - -import com.google.protobuf.Service; -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.hadoop.hbase.CoprocessorEnvironment; -import org.apache.hadoop.hbase.HBaseIOException; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.MasterNotRunningException; -import org.apache.hadoop.hbase.NamespaceDescriptor; -import org.apache.hadoop.hbase.PleaseHoldException; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.client.SnapshotDescription; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.constraint.ConstraintException; -import org.apache.hadoop.hbase.coprocessor.CoreCoprocessor; -import org.apache.hadoop.hbase.coprocessor.HasMasterServices; -import org.apache.hadoop.hbase.coprocessor.MasterCoprocessor; -import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; -import org.apache.hadoop.hbase.coprocessor.MasterObserver; -import org.apache.hadoop.hbase.coprocessor.ObserverContext; -import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.net.Address; -import org.apache.hadoop.hbase.security.UserProvider; -import org.apache.hadoop.hbase.security.access.AccessChecker; -import org.apache.yetus.audience.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; -import org.apache.hbase.thirdparty.com.google.common.collect.Sets; - -// TODO: Encapsulate MasterObserver functions into separate subclass. -@CoreCoprocessor -@InterfaceAudience.Private -public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver { - static final Logger LOG = LoggerFactory.getLogger(RSGroupAdminEndpoint.class); - - private MasterServices master; - // Only instance of RSGroupInfoManager. RSGroup aware load balancers ask for this instance on - // their setup. - private RSGroupInfoManager groupInfoManager; - private RSGroupAdminServer groupAdminServer; - private RSGroupAdminServiceImpl groupAdminService = new RSGroupAdminServiceImpl(); - - @Override - public void start(CoprocessorEnvironment env) throws IOException { - if (!(env instanceof HasMasterServices)) { - throw new IOException("Does not implement HMasterServices"); - } - - master = ((HasMasterServices) env).getMasterServices(); - groupInfoManager = RSGroupInfoManagerImpl.getInstance(master); - groupAdminServer = new RSGroupAdminServer(master, groupInfoManager); - Class clazz = - master.getConfiguration().getClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, null); - if (!RSGroupableBalancer.class.isAssignableFrom(clazz)) { - throw new IOException("Configured balancer does not support RegionServer groups."); - } - AccessChecker accessChecker = ((HasMasterServices) env).getMasterServices().getAccessChecker(); - - // set the user-provider. - UserProvider userProvider = UserProvider.instantiate(env.getConfiguration()); - groupAdminService.initialize(master, groupAdminServer, accessChecker, userProvider); - } - - @Override - public void stop(CoprocessorEnvironment env) { - } - - @Override - public Iterable getServices() { - return Collections.singleton(groupAdminService); - } - - @Override - public Optional getMasterObserver() { - return Optional.of(this); - } - - RSGroupInfoManager getGroupInfoManager() { - return groupInfoManager; - } - - @VisibleForTesting - RSGroupAdminServiceImpl getGroupAdminService() { - return groupAdminService; - } - - private void assignTableToGroup(TableDescriptor desc) throws IOException { - String groupName = - master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString()) - .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); - if (groupName == null) { - groupName = RSGroupInfo.DEFAULT_GROUP; - } - RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); - if (rsGroupInfo == null) { - throw new ConstraintException( - "Default RSGroup (" + groupName + ") for this table's namespace does not exist."); - } - if (!rsGroupInfo.containsTable(desc.getTableName())) { - LOG.debug("Pre-moving table " + desc.getTableName() + " to RSGroup " + groupName); - groupAdminServer.moveTables(Sets.newHashSet(desc.getTableName()), groupName); - } - } - - ///////////////////////////////////////////////////////////////////////////// - // MasterObserver overrides - ///////////////////////////////////////////////////////////////////////////// - - private boolean rsgroupHasServersOnline(TableDescriptor desc) throws IOException { - String groupName; - try { - groupName = master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString()) - .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); - if (groupName == null) { - groupName = RSGroupInfo.DEFAULT_GROUP; - } - } catch (MasterNotRunningException | PleaseHoldException e) { - LOG.info("Master has not initialized yet; temporarily using default RSGroup '" + - RSGroupInfo.DEFAULT_GROUP + "' for deploy of system table"); - groupName = RSGroupInfo.DEFAULT_GROUP; - } - - RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); - if (rsGroupInfo == null) { - throw new ConstraintException( - "Default RSGroup (" + groupName + ") for this table's " + "namespace does not exist."); - } - - for (ServerName onlineServer : master.getServerManager().createDestinationServersList()) { - if (rsGroupInfo.getServers().contains(onlineServer.getAddress())) { - return true; - } - } - return false; - } - - @Override - public void preCreateTableAction(final ObserverContext ctx, - final TableDescriptor desc, final RegionInfo[] regions) throws IOException { - if (!desc.getTableName().isSystemTable() && !rsgroupHasServersOnline(desc)) { - throw new HBaseIOException("No online servers in the rsgroup, which table " + - desc.getTableName().getNameAsString() + " belongs to"); - } - } - - // Assign table to default RSGroup. - @Override - public void postCreateTable(ObserverContext ctx, - TableDescriptor desc, RegionInfo[] regions) throws IOException { - assignTableToGroup(desc); - } - - // Remove table from its RSGroup. - @Override - public void postDeleteTable(ObserverContext ctx, - TableName tableName) throws IOException { - try { - RSGroupInfo group = groupAdminServer.getRSGroupInfoOfTable(tableName); - if (group != null) { - LOG.debug(String.format("Removing deleted table '%s' from rsgroup '%s'", tableName, - group.getName())); - groupAdminServer.moveTables(Sets.newHashSet(tableName), null); - } - } catch (IOException ex) { - LOG.debug("Failed to perform RSGroup information cleanup for table: " + tableName, ex); - } - } - - @Override - public void preCreateNamespace(ObserverContext ctx, - NamespaceDescriptor ns) throws IOException { - String group = ns.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); - if (group != null && groupAdminServer.getRSGroupInfo(group) == null) { - throw new ConstraintException("Region server group " + group + " does not exit"); - } - } - - @Override - public void preModifyNamespace(ObserverContext ctx, - NamespaceDescriptor currentNsDesc, NamespaceDescriptor newNsDesc) throws IOException { - preCreateNamespace(ctx, newNsDesc); - } - - @Override - public void preCloneSnapshot(ObserverContext ctx, - SnapshotDescription snapshot, TableDescriptor desc) throws IOException { - assignTableToGroup(desc); - } - - @Override - public void postClearDeadServers(ObserverContext ctx, - List servers, List notClearedServers) throws IOException { - Set

clearedServer = - servers.stream().filter(server -> !notClearedServers.contains(server)) - .map(ServerName::getAddress).collect(Collectors.toSet()); - if (!clearedServer.isEmpty()) { - groupAdminServer.removeServers(clearedServer); - } - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServiceImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServiceImpl.java deleted file mode 100644 index 918a4fead8a..00000000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServiceImpl.java +++ /dev/null @@ -1,378 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.rsgroup; - -import com.google.protobuf.RpcCallback; -import com.google.protobuf.RpcController; -import java.io.IOException; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; -import org.apache.hadoop.hbase.ipc.RpcServer; -import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.net.Address; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.AddRSGroupRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.AddRSGroupResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfTableRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfTableResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersAndTablesRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersAndTablesResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveServersResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveTablesRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveTablesResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupResponse; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveServersRequest; -import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveServersResponse; -import org.apache.hadoop.hbase.security.User; -import org.apache.hadoop.hbase.security.UserProvider; -import org.apache.hadoop.hbase.security.access.AccessChecker; -import org.apache.hadoop.hbase.security.access.Permission.Action; - -import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; -import org.apache.hbase.thirdparty.com.google.common.collect.Sets; - -/** - * Implementation of RSGroupAdminService defined in RSGroupAdmin.proto. This class calls - * {@link RSGroupAdminServer} for actual work, converts result to protocol buffer response, handles - * exceptions if any occurred and then calls the {@code RpcCallback} with the response. - */ -class RSGroupAdminServiceImpl extends RSGroupAdminProtos.RSGroupAdminService { - - private MasterServices master; - - private RSGroupAdminServer groupAdminServer; - - private AccessChecker accessChecker; - - /** Provider for mapping principal names to Users */ - private UserProvider userProvider; - - RSGroupAdminServiceImpl() { - } - - void initialize(MasterServices master, RSGroupAdminServer groupAdminServer, - AccessChecker accessChecker, UserProvider userProvider) { - this.master = master; - this.groupAdminServer = groupAdminServer; - this.accessChecker = accessChecker; - this.userProvider = userProvider; - } - - @VisibleForTesting - void checkPermission(String request) throws IOException { - accessChecker.requirePermission(getActiveUser(), request, null, Action.ADMIN); - } - - /** - * Returns the active user to which authorization checks should be applied. If we are in the - * context of an RPC call, the remote user is used, otherwise the currently logged in user is - * used. - */ - private User getActiveUser() throws IOException { - // for non-rpc handling, fallback to system user - Optional optionalUser = RpcServer.getRequestUser(); - if (optionalUser.isPresent()) { - return optionalUser.get(); - } - return userProvider.getCurrent(); - } - - @Override - public void getRSGroupInfo(RpcController controller, GetRSGroupInfoRequest request, - RpcCallback done) { - GetRSGroupInfoResponse.Builder builder = GetRSGroupInfoResponse.newBuilder(); - String groupName = request.getRSGroupName(); - RSGroupAdminEndpoint.LOG.info( - master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, group=" + groupName); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preGetRSGroupInfo(groupName); - } - checkPermission("getRSGroupInfo"); - RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); - if (rsGroupInfo != null) { - builder.setRSGroupInfo(ProtobufUtil.toProtoGroupInfo(rsGroupInfo)); - } - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postGetRSGroupInfo(groupName); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void getRSGroupInfoOfTable(RpcController controller, GetRSGroupInfoOfTableRequest request, - RpcCallback done) { - GetRSGroupInfoOfTableResponse.Builder builder = GetRSGroupInfoOfTableResponse.newBuilder(); - TableName tableName = ProtobufUtil.toTableName(request.getTableName()); - RSGroupAdminEndpoint.LOG.info( - master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, table=" + tableName); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preGetRSGroupInfoOfTable(tableName); - } - checkPermission("getRSGroupInfoOfTable"); - RSGroupInfo RSGroupInfo = groupAdminServer.getRSGroupInfoOfTable(tableName); - if (RSGroupInfo != null) { - builder.setRSGroupInfo(ProtobufUtil.toProtoGroupInfo(RSGroupInfo)); - } - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postGetRSGroupInfoOfTable(tableName); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void moveServers(RpcController controller, MoveServersRequest request, - RpcCallback done) { - MoveServersResponse.Builder builder = MoveServersResponse.newBuilder(); - Set
hostPorts = Sets.newHashSet(); - for (HBaseProtos.ServerName el : request.getServersList()) { - hostPorts.add(Address.fromParts(el.getHostName(), el.getPort())); - } - RSGroupAdminEndpoint.LOG.info(master.getClientIdAuditPrefix() + " move servers " + hostPorts + - " to rsgroup " + request.getTargetGroup()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preMoveServers(hostPorts, request.getTargetGroup()); - } - checkPermission("moveServers"); - groupAdminServer.moveServers(hostPorts, request.getTargetGroup()); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postMoveServers(hostPorts, request.getTargetGroup()); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void moveTables(RpcController controller, MoveTablesRequest request, - RpcCallback done) { - MoveTablesResponse.Builder builder = MoveTablesResponse.newBuilder(); - Set tables = new HashSet<>(request.getTableNameList().size()); - for (HBaseProtos.TableName tableName : request.getTableNameList()) { - tables.add(ProtobufUtil.toTableName(tableName)); - } - RSGroupAdminEndpoint.LOG.info(master.getClientIdAuditPrefix() + " move tables " + tables + - " to rsgroup " + request.getTargetGroup()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preMoveTables(tables, request.getTargetGroup()); - } - checkPermission("moveTables"); - groupAdminServer.moveTables(tables, request.getTargetGroup()); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postMoveTables(tables, request.getTargetGroup()); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void addRSGroup(RpcController controller, AddRSGroupRequest request, - RpcCallback done) { - AddRSGroupResponse.Builder builder = AddRSGroupResponse.newBuilder(); - RSGroupAdminEndpoint.LOG - .info(master.getClientIdAuditPrefix() + " add rsgroup " + request.getRSGroupName()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preAddRSGroup(request.getRSGroupName()); - } - checkPermission("addRSGroup"); - groupAdminServer.addRSGroup(request.getRSGroupName()); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postAddRSGroup(request.getRSGroupName()); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void removeRSGroup(RpcController controller, RemoveRSGroupRequest request, - RpcCallback done) { - RemoveRSGroupResponse.Builder builder = RemoveRSGroupResponse.newBuilder(); - RSGroupAdminEndpoint.LOG - .info(master.getClientIdAuditPrefix() + " remove rsgroup " + request.getRSGroupName()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preRemoveRSGroup(request.getRSGroupName()); - } - checkPermission("removeRSGroup"); - groupAdminServer.removeRSGroup(request.getRSGroupName()); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postRemoveRSGroup(request.getRSGroupName()); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void balanceRSGroup(RpcController controller, BalanceRSGroupRequest request, - RpcCallback done) { - BalanceRSGroupResponse.Builder builder = BalanceRSGroupResponse.newBuilder(); - RSGroupAdminEndpoint.LOG.info( - master.getClientIdAuditPrefix() + " balance rsgroup, group=" + request.getRSGroupName()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preBalanceRSGroup(request.getRSGroupName()); - } - checkPermission("balanceRSGroup"); - boolean balancerRan = groupAdminServer.balanceRSGroup(request.getRSGroupName()); - builder.setBalanceRan(balancerRan); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postBalanceRSGroup(request.getRSGroupName(), balancerRan); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - builder.setBalanceRan(false); - } - done.run(builder.build()); - } - - @Override - public void listRSGroupInfos(RpcController controller, ListRSGroupInfosRequest request, - RpcCallback done) { - ListRSGroupInfosResponse.Builder builder = ListRSGroupInfosResponse.newBuilder(); - RSGroupAdminEndpoint.LOG.info(master.getClientIdAuditPrefix() + " list rsgroup"); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preListRSGroups(); - } - checkPermission("listRSGroup"); - for (RSGroupInfo RSGroupInfo : groupAdminServer.listRSGroups()) { - builder.addRSGroupInfo(ProtobufUtil.toProtoGroupInfo(RSGroupInfo)); - } - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postListRSGroups(); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void getRSGroupInfoOfServer(RpcController controller, - GetRSGroupInfoOfServerRequest request, RpcCallback done) { - GetRSGroupInfoOfServerResponse.Builder builder = GetRSGroupInfoOfServerResponse.newBuilder(); - Address hp = - Address.fromParts(request.getServer().getHostName(), request.getServer().getPort()); - RSGroupAdminEndpoint.LOG - .info(master.getClientIdAuditPrefix() + " initiates rsgroup info retrieval, server=" + hp); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preGetRSGroupInfoOfServer(hp); - } - checkPermission("getRSGroupInfoOfServer"); - RSGroupInfo info = groupAdminServer.getRSGroupOfServer(hp); - if (info != null) { - builder.setRSGroupInfo(ProtobufUtil.toProtoGroupInfo(info)); - } - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postGetRSGroupInfoOfServer(hp); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void moveServersAndTables(RpcController controller, MoveServersAndTablesRequest request, - RpcCallback done) { - MoveServersAndTablesResponse.Builder builder = MoveServersAndTablesResponse.newBuilder(); - Set
hostPorts = Sets.newHashSet(); - for (HBaseProtos.ServerName el : request.getServersList()) { - hostPorts.add(Address.fromParts(el.getHostName(), el.getPort())); - } - Set tables = new HashSet<>(request.getTableNameList().size()); - for (HBaseProtos.TableName tableName : request.getTableNameList()) { - tables.add(ProtobufUtil.toTableName(tableName)); - } - RSGroupAdminEndpoint.LOG.info(master.getClientIdAuditPrefix() + " move servers " + hostPorts + - " and tables " + tables + " to rsgroup" + request.getTargetGroup()); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preMoveServersAndTables(hostPorts, tables, - request.getTargetGroup()); - } - checkPermission("moveServersAndTables"); - groupAdminServer.moveServersAndTables(hostPorts, tables, request.getTargetGroup()); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postMoveServersAndTables(hostPorts, tables, - request.getTargetGroup()); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } - - @Override - public void removeServers(RpcController controller, RemoveServersRequest request, - RpcCallback done) { - RemoveServersResponse.Builder builder = RemoveServersResponse.newBuilder(); - Set
servers = Sets.newHashSet(); - for (HBaseProtos.ServerName el : request.getServersList()) { - servers.add(Address.fromParts(el.getHostName(), el.getPort())); - } - RSGroupAdminEndpoint.LOG.info( - master.getClientIdAuditPrefix() + " remove decommissioned servers from rsgroup: " + servers); - try { - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().preRemoveServers(servers); - } - checkPermission("removeServers"); - groupAdminServer.removeServers(servers); - if (master.getMasterCoprocessorHost() != null) { - master.getMasterCoprocessorHost().postRemoveServers(servers); - } - } catch (IOException e) { - CoprocessorRpcUtils.setControllerException(controller, e); - } - done.run(builder.build()); - } -} \ No newline at end of file diff --git a/hbase-shell/pom.xml b/hbase-shell/pom.xml index a6b62ad5895..7483da5968e 100644 --- a/hbase-shell/pom.xml +++ b/hbase-shell/pom.xml @@ -187,6 +187,41 @@ + + rsgroup + + + !skip-rsgroup + + + + + org.apache.hbase + hbase-rsgroup + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-test-source + + add-test-source + + + + src/test/rsgroup + + + + + + + + skipShellTests diff --git a/pom.xml b/pom.xml index ec6437aa263..014d31d912a 100755 --- a/pom.xml +++ b/pom.xml @@ -1540,6 +1540,7 @@ hbase-procedure-${project.version}-tests.jar hbase-it-${project.version}-tests.jar hbase-annotations-${project.version}-tests.jar + hbase-rsgroup-${project.version}-tests.jar hbase-mapreduce-${project.version}-tests.jar hbase-zookeeper-${project.version}-tests.jar bash @@ -1669,6 +1670,18 @@ test-jar test + + hbase-rsgroup + org.apache.hbase + ${project.version} + + + hbase-rsgroup + org.apache.hbase + ${project.version} + test-jar + test + hbase-replication org.apache.hbase @@ -2302,6 +2315,17 @@ --> + + rsgroup + + + !skip-rsgroup + + + + hbase-rsgroup + + build-with-jdk8