Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-23276 Add admin methods to get tables within a group (#1118) 

Signed-off-by: Guanghao Zhang <zghao@apache.org>
This commit is contained in:
Duo Zhang 2020-02-06 19:53:57 +08:00 committed by Duo Zhang
parent 5de0a5ef7e
commit 7386369fec
19 changed files with 402 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
View File

@ -63,6 +63,7 @@ import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;
/**
@ -2322,6 +2323,30 @@ public interface Admin extends Abortable, Closeable {
*/
List<RSGroupInfo> listRSGroups() throws IOException;
/**
* Get all tables in this RegionServer group.
* @param groupName the group name
* @throws IOException if a remote or network exception occurs
* @see #getConfiguredNamespacesAndTablesInRSGroup(String)
*/
List<TableName> listTablesInRSGroup(String groupName) throws IOException;
/**
* Get the namespaces and tables which have this RegionServer group in descriptor.
* <p/>
* The difference between this method and {@link #listTablesInRSGroup(String)} is that, this
* method will not include the table which is actually in this RegionServr group but without the
* RegionServer group configuration in its {@link TableDescriptor}. For example, we have a group
* 'A', and we make namespace 'nsA' in this group, then all the tables under this namespace will
* in the group 'A', but this method will not return these tables but only the namespace 'nsA',
* while the {@link #listTablesInRSGroup(String)} will return all these tables.
* @param groupName the group name
* @throws IOException if a remote or network exception occurs
* @see #listTablesInRSGroup(String)
*/
Pair<List<String>, List<TableName>> getConfiguredNamespacesAndTablesInRSGroup(String groupName)
throws IOException;
/**
* Remove RegionServer group associated with the given name
* @param groupName the group name

13
hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java
View File

@ -66,6 +66,7 @@ import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -1004,6 +1005,17 @@ class AdminOverAsyncAdmin implements Admin {
return get(admin.listRSGroups());
}
@Override
public List<TableName> listTablesInRSGroup(String groupName) throws IOException {
return get(admin.listTablesInRSGroup(groupName));
}
@Override
public Pair<List<String>, List<TableName>>
getConfiguredNamespacesAndTablesInRSGroup(String groupName) throws IOException {
return get(admin.getConfiguredNamespacesAndTablesInRSGroup(groupName));
}
@Override
public RSGroupInfo getRSGroup(Address hostPort) throws IOException {
return get(admin.getRSGroup(hostPort));
@ -1023,5 +1035,4 @@ class AdminOverAsyncAdmin implements Admin {
public void setRSGroup(Set<TableName> tables, String groupName) throws IOException {
get(admin.setRSGroup(tables, groupName));
}
}

25
hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
View File

@ -51,6 +51,7 @@ import org.apache.hadoop.hbase.rsgroup.RSGroupInfo;
import org.apache.hadoop.hbase.security.access.GetUserPermissionsRequest;
import org.apache.hadoop.hbase.security.access.Permission;
import org.apache.hadoop.hbase.security.access.UserPermission;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;
/**
@ -1564,6 +1565,30 @@ public interface AsyncAdmin {
*/
CompletableFuture<List<RSGroupInfo>> listRSGroups();
/**
* Get all tables in this RegionServer group.
* @param groupName the group name
* @throws IOException if a remote or network exception occurs
* @see #getConfiguredNamespacesAndTablesInRSGroup(String)
*/
CompletableFuture<List<TableName>> listTablesInRSGroup(String groupName);
/**
* Get the namespaces and tables which have this RegionServer group in descriptor.
* <p/>
* The difference between this method and {@link #listTablesInRSGroup(String)} is that, this
* method will not include the table which is actually in this RegionServr group but without the
* RegionServer group configuration in its {@link TableDescriptor}. For example, we have a group
* 'A', and we make namespace 'nsA' in this group, then all the tables under this namespace will
* in the group 'A', but this method will not return these tables but only the namespace 'nsA',
* while the {@link #listTablesInRSGroup(String)} will return all these tables.
* @param groupName the group name
* @throws IOException if a remote or network exception occurs
* @see #listTablesInRSGroup(String)
*/
CompletableFuture<Pair<List<String>, List<TableName>>>
getConfiguredNamespacesAndTablesInRSGroup(String groupName);
/**
* Remove RegionServer group associated with the given name
* @param groupName the group name

12
hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
View File

@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.security.access.GetUserPermissionsRequest;
import org.apache.hadoop.hbase.security.access.Permission;
import org.apache.hadoop.hbase.security.access.UserPermission;
import org.apache.hadoop.hbase.util.FutureUtils;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;
/**
@ -882,6 +883,17 @@ class AsyncHBaseAdmin implements AsyncAdmin {
return wrap(rawAdmin.listRSGroups());
}
@Override
public CompletableFuture<List<TableName>> listTablesInRSGroup(String groupName) {
return wrap(rawAdmin.listTablesInRSGroup(groupName));
}
@Override
public CompletableFuture<Pair<List<String>, List<TableName>>>
getConfiguredNamespacesAndTablesInRSGroup(String groupName) {
return wrap(rawAdmin.getConfiguredNamespacesAndTablesInRSGroup(groupName));
}
@Override
public CompletableFuture<RSGroupInfo> getRSGroup(Address hostPort) {
return wrap(rawAdmin.getRSGroup(hostPort));

33
hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
View File

@ -98,6 +98,7 @@ import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -289,6 +290,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.AddR
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.AddRSGroupResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.GetConfiguredNamespacesAndTablesInRSGroupRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.GetConfiguredNamespacesAndTablesInRSGroupResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfTableRequest;
@ -297,6 +300,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.GetR
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.ListTablesInRSGroupRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.ListTablesInRSGroupResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.MoveServersRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.MoveServersResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupRequest;
@ -335,6 +340,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
*/
@InterfaceAudience.Private
class RawAsyncHBaseAdmin implements AsyncAdmin {
public static final String FLUSH_TABLE_PROCEDURE_SIGNATURE = "flush-table-proc";
private static final Logger LOG = LoggerFactory.getLogger(AsyncHBaseAdmin.class);
@ -4002,6 +4008,33 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
);
}
@Override
public CompletableFuture<List<TableName>> listTablesInRSGroup(String groupName) {
return this.<List<TableName>> newMasterCaller()
.action((controller, stub) -> this
.<ListTablesInRSGroupRequest, ListTablesInRSGroupResponse, List<TableName>> call(controller,
stub, ListTablesInRSGroupRequest.newBuilder().setGroupName(groupName).build(),
(s, c, req, done) -> s.listTablesInRSGroup(c, req, done), resp -> resp.getTableNameList()
.stream().map(ProtobufUtil::toTableName).collect(Collectors.toList())))
.call();
}
@Override
public CompletableFuture<Pair<List<String>, List<TableName>>>
getConfiguredNamespacesAndTablesInRSGroup(String groupName) {
return this.<Pair<List<String>, List<TableName>>> newMasterCaller()
.action((controller, stub) -> this
.<GetConfiguredNamespacesAndTablesInRSGroupRequest,
GetConfiguredNamespacesAndTablesInRSGroupResponse,
Pair<List<String>, List<TableName>>> call(controller, stub,
GetConfiguredNamespacesAndTablesInRSGroupRequest.newBuilder().setGroupName(groupName)
.build(),
(s, c, req, done) -> s.getConfiguredNamespacesAndTablesInRSGroup(c, req, done),
resp -> Pair.newPair(resp.getNamespaceList(), resp.getTableNameList().stream()
.map(ProtobufUtil::toTableName).collect(Collectors.toList()))))
.call();
}
@Override
public CompletableFuture<RSGroupInfo> getRSGroup(Address hostPort) {
return this.<RSGroupInfo> newMasterCaller()

5
hbase-it/src/test/java/org/apache/hadoop/hbase/rsgroup/IntegrationTestRSGroup.java
View File

@ -22,7 +22,6 @@ package org.apache.hadoop.hbase.rsgroup;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.IntegrationTestingUtility;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.testclassification.IntegrationTests;
import org.junit.After;
import org.junit.Before;
@ -47,13 +46,11 @@ public class IntegrationTestRSGroup extends TestRSGroupsBase {
TEST_UTIL = new IntegrationTestingUtility();
TEST_UTIL.getConfiguration().set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
RSGroupBasedLoadBalancer.class.getName());
TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
RSGroupAdminEndpoint.class.getName());
TEST_UTIL.getConfiguration().setBoolean(RSGroupInfoManager.RS_GROUP_ENABLED, true);
((IntegrationTestingUtility) TEST_UTIL).initializeCluster(NUM_SLAVES_BASE);
// set shared configs
ADMIN = TEST_UTIL.getAdmin();
CLUSTER = TEST_UTIL.getHBaseClusterInterface();
RS_GROUP_ADMIN_CLIENT = new RSGroupAdminClient(TEST_UTIL.getConnection());
LOG.info("Done initializing cluster");
initialized = true;
// cluster may not be clean

6
hbase-protocol-shaded/src/main/protobuf/Master.proto
View File

@ -1113,6 +1113,12 @@ service MasterService {
rpc RemoveServers(RemoveServersRequest)
returns (RemoveServersResponse);
rpc ListTablesInRSGroup(ListTablesInRSGroupRequest)
returns (ListTablesInRSGroupResponse);
rpc GetConfiguredNamespacesAndTablesInRSGroup(GetConfiguredNamespacesAndTablesInRSGroupRequest)
returns (GetConfiguredNamespacesAndTablesInRSGroupResponse);
}
// HBCK Service definitions.

17
hbase-protocol-shaded/src/main/protobuf/RSGroupAdmin.proto
View File

@ -122,6 +122,23 @@ message RemoveServersRequest {
message RemoveServersResponse {
}
message ListTablesInRSGroupRequest {
required string group_name = 1;
}
message ListTablesInRSGroupResponse {
repeated TableName table_name = 1;
}
message GetConfiguredNamespacesAndTablesInRSGroupRequest {
required string group_name = 1;
}
message GetConfiguredNamespacesAndTablesInRSGroupResponse {
repeated string namespace = 1;
repeated TableName table_name = 2;
}
service RSGroupAdminService {
rpc GetRSGroupInfo(GetRSGroupInfoRequest)
returns (GetRSGroupInfoResponse);

34
hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
View File

@ -1324,6 +1324,40 @@ public interface MasterObserver {
default void postListRSGroups(final ObserverContext<MasterCoprocessorEnvironment> ctx)
throws IOException {}
/**
* Called before listing all tables in the region server group.
* @param ctx the environment to interact with the framework and master
* @param groupName name of the region server group
*/
default void preListTablesInRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final String groupName) throws IOException {}
/**
* Called after listing all tables in the region server group.
* @param ctx the environment to interact with the framework and master
* @param groupName name of the region server group
*/
default void postListTablesInRSGroup(final ObserverContext<MasterCoprocessorEnvironment> ctx,
final String groupName) throws IOException {}
/**
* Called before getting the configured namespaces and tables in the region server group.
* @param ctx the environment to interact with the framework and master
* @param groupName name of the region server group
*/
default void preGetConfiguredNamespacesAndTablesInRSGroup(
final ObserverContext<MasterCoprocessorEnvironment> ctx, final String groupName)
throws IOException {}
/**
* Called after getting the configured namespaces and tables in the region server group.
* @param ctx the environment to interact with the framework and master
* @param groupName name of the region server group
*/
default void postGetConfiguredNamespacesAndTablesInRSGroup(
final ObserverContext<MasterCoprocessorEnvironment> ctx, final String groupName)
throws IOException {}
/**
* Called before getting region server group info of the passed server.
* @param ctx the environment to interact with the framework and master

42
hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
View File

@ -1529,6 +1529,48 @@ public class MasterCoprocessorHost
});
}
public void preListTablesInRSGroup(final String groupName) throws IOException {
execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
@Override
protected void call(MasterObserver observer) throws IOException {
observer.preListTablesInRSGroup(this, groupName);
}
});
}
public void postListTablesInRSGroup(final String groupName) throws IOException {
execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
@Override
protected void call(MasterObserver observer) throws IOException {
observer.postListTablesInRSGroup(this, groupName);
}
});
}
public void preGetConfiguredNamespacesAndTablesInRSGroup(final String groupName)
throws IOException {
execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
@Override
protected void call(MasterObserver observer) throws IOException {
observer.preGetConfiguredNamespacesAndTablesInRSGroup(this, groupName);
}
});
}
public void postGetConfiguredNamespacesAndTablesInRSGroup(final String groupName)
throws IOException {
execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
@Override
protected void call(MasterObserver observer) throws IOException {
observer.postGetConfiguredNamespacesAndTablesInRSGroup(this, groupName);
}
});
}
public void preGetRSGroupInfoOfServer(final Address server) throws IOException {
execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
@Override

64
hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
View File

@ -96,6 +96,7 @@ import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerDescription;
import org.apache.hadoop.hbase.rsgroup.RSGroupInfo;
import org.apache.hadoop.hbase.rsgroup.RSGroupUtil;
import org.apache.hadoop.hbase.security.Superusers;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.access.AccessChecker;
@ -321,6 +322,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.AddR
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.AddRSGroupResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.GetConfiguredNamespacesAndTablesInRSGroupRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.GetConfiguredNamespacesAndTablesInRSGroupResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfTableRequest;
@ -329,6 +332,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.GetR
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.ListTablesInRSGroupRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.ListTablesInRSGroupResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.MoveServersRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.MoveServersResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupRequest;
@ -3161,4 +3166,63 @@ public class MasterRpcServices extends RSRpcServices implements
}
return builder.build();
}
@Override
public ListTablesInRSGroupResponse listTablesInRSGroup(RpcController controller,
ListTablesInRSGroupRequest request) throws ServiceException {
ListTablesInRSGroupResponse.Builder builder = ListTablesInRSGroupResponse.newBuilder();
String groupName = request.getGroupName();
LOG.info(master.getClientIdAuditPrefix() + " list tables in rsgroup " + groupName);
try {
if (master.getMasterCoprocessorHost() != null) {
master.getMasterCoprocessorHost().preListTablesInRSGroup(groupName);
}
boolean isDefaultGroup = RSGroupInfo.DEFAULT_GROUP.equals(groupName);
for (TableDescriptor td : master.getTableDescriptors().getAll().values()) {
// no config means in default group
if (RSGroupUtil.getRSGroupInfo(master, master.getRSGroupInfoManager(), td.getTableName())
.map(g -> g.getName().equals(groupName)).orElse(isDefaultGroup)) {
builder.addTableName(ProtobufUtil.toProtoTableName(td.getTableName()));
}
}
if (master.getMasterCoprocessorHost() != null) {
master.getMasterCoprocessorHost().postListTablesInRSGroup(groupName);
}
} catch (IOException e) {
throw new ServiceException(e);
}
return builder.build();
}
@Override
public GetConfiguredNamespacesAndTablesInRSGroupResponse
getConfiguredNamespacesAndTablesInRSGroup(RpcController controller,
GetConfiguredNamespacesAndTablesInRSGroupRequest request) throws ServiceException {
GetConfiguredNamespacesAndTablesInRSGroupResponse.Builder builder =
GetConfiguredNamespacesAndTablesInRSGroupResponse.newBuilder();
String groupName = request.getGroupName();
LOG.info(master.getClientIdAuditPrefix() + " get configured namespaces and tables in rsgroup " +
groupName);
try {
if (master.getMasterCoprocessorHost() != null) {
master.getMasterCoprocessorHost().preGetConfiguredNamespacesAndTablesInRSGroup(groupName);
}
for (NamespaceDescriptor nd : master.getClusterSchema().getNamespaces()) {
if (groupName.equals(nd.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP))) {
builder.addNamespace(nd.getName());
}
}
for (TableDescriptor td : master.getTableDescriptors().getAll().values()) {
if (td.getRegionServerGroup().map(g -> g.equals(groupName)).orElse(false)) {
builder.addTableName(ProtobufUtil.toProtoTableName(td.getTableName()));
}
}
if (master.getMasterCoprocessorHost() != null) {
master.getMasterCoprocessorHost().postGetConfiguredNamespacesAndTablesInRSGroup(groupName);
}
} catch (IOException e) {
throw new ServiceException(e);
}
return builder.build();
}
}

14
hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
View File

@ -2660,6 +2660,20 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
null, Permission.Action.ADMIN);
}
@Override
public void preListTablesInRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx,
String groupName) throws IOException {
accessChecker.requirePermission(getActiveUser(ctx), "listTablesInRSGroup",
null, Permission.Action.ADMIN);
}
@Override
public void preGetConfiguredNamespacesAndTablesInRSGroup(
ObserverContext<MasterCoprocessorEnvironment> ctx, String groupName) throws IOException {
accessChecker.requirePermission(getActiveUser(ctx), "getConfiguredNamespacesAndTablesInRSGroup",
null, Permission.Action.ADMIN);
}
@Override
public void preGetRSGroupInfoOfServer(ObserverContext<MasterCoprocessorEnvironment> ctx,
Address server) throws IOException {

21
hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestMigrateRSGroupInfo.java
View File

@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos;
@ -42,8 +43,8 @@ import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RSGroupTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.zookeeper.KeeperException;
import org.junit.After;
import org.junit.Before;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@ -64,18 +65,25 @@ public class TestMigrateRSGroupInfo extends TestRSGroupsBase {
private static byte[] FAMILY = Bytes.toBytes("family");
@Before
public void setUp() throws Exception {
private static RSGroupAdminClient RS_GROUP_ADMIN_CLIENT;
@BeforeClass
public static void setUp() throws Exception {
TEST_UTIL.getConfiguration().setClass(HConstants.MASTER_IMPL, HMasterForTest.class,
HMaster.class);
// confirm that we could enable rs group by setting the old CP.
TEST_UTIL.getConfiguration().setBoolean(RSGroupInfoManager.RS_GROUP_ENABLED, false);
TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
RSGroupAdminEndpoint.class.getName());
setUpTestBeforeClass();
RS_GROUP_ADMIN_CLIENT = new RSGroupAdminClient(TEST_UTIL.getConnection());
for (int i = 0; i < NUM_TABLES; i++) {
TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME_PREFIX + i), FAMILY);
}
}
@After
public void tearDown() throws Exception {
@AfterClass
public static void tearDown() throws Exception {
tearDownAfterClass();
}
@ -124,6 +132,7 @@ public class TestMigrateRSGroupInfo extends TestRSGroupsBase {
RESUME = new CountDownLatch(1);
TEST_UTIL.getMiniHBaseCluster().startMaster();
TEST_UTIL.invalidateConnection();
RS_GROUP_ADMIN_CLIENT = new RSGroupAdminClient(TEST_UTIL.getConnection());
// wait until we can get the rs group info for a table
TEST_UTIL.waitFor(30000, () -> {

30
hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java
View File

@ -27,7 +27,6 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.ServerName;
@ -186,7 +185,7 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase {
addGroup("bar", 3);
TEST_UTIL.createTable(tableName, Bytes.toBytes("f"));
ADMIN.setRSGroup(Sets.newHashSet(tableName), "bar");
RSGroupInfo barGroup = RS_GROUP_ADMIN_CLIENT.getRSGroupInfo("bar");
RSGroupInfo barGroup = ADMIN.getRSGroup("bar");
// group is not empty therefore it should fail
try {
ADMIN.removeRSGroup(barGroup.getName());
@ -200,7 +199,7 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase {
} catch (IOException e) {
}
ADMIN.setRSGroup(barGroup.getTables(), RSGroupInfo.DEFAULT_GROUP);
ADMIN.setRSGroup(Sets.newHashSet(ADMIN.listTablesInRSGroup("bar")), RSGroupInfo.DEFAULT_GROUP);
try {
ADMIN.removeRSGroup(barGroup.getName());
fail("Expected move servers to fail");
@ -257,12 +256,12 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase {
// verify tables' not exist in old group
Set<TableName> defaultTables =
RS_GROUP_ADMIN_CLIENT.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables();
Sets.newHashSet(ADMIN.listTablesInRSGroup(RSGroupInfo.DEFAULT_GROUP));
assertFalse(defaultTables.contains(tableNameA));
assertFalse(defaultTables.contains(tableNameB));
// verify tables' exist in new group
Set<TableName> newGroupTables = RS_GROUP_ADMIN_CLIENT.getRSGroupInfo(newGroupName).getTables();
Set<TableName> newGroupTables = Sets.newHashSet(ADMIN.listTablesInRSGroup(newGroupName));
assertTrue(newGroupTables.contains(tableNameA));
assertTrue(newGroupTables.contains(tableNameB));
}
@ -316,13 +315,13 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase {
// test truncate
ADMIN.disableTable(tableName);
ADMIN.truncateTable(tableName, true);
assertEquals(1, RS_GROUP_ADMIN_CLIENT.getRSGroupInfo(newGroup.getName()).getTables().size());
assertEquals(tableName,
RS_GROUP_ADMIN_CLIENT.getRSGroupInfo(newGroup.getName()).getTables().first());
List<TableName> tablesInGroup = ADMIN.listTablesInRSGroup(newGroup.getName());
assertEquals(1, tablesInGroup.size());
assertEquals(tableName, tablesInGroup.get(0));
// verify removed table is removed from group
TEST_UTIL.deleteTable(tableName);
assertEquals(0, RS_GROUP_ADMIN_CLIENT.getRSGroupInfo(newGroup.getName()).getTables().size());
assertEquals(0, ADMIN.listTablesInRSGroup(newGroup.getName()).size());
}
@Test
@ -418,14 +417,16 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase {
assertTrue("Constraint not violated for table " + tableDescTwo.getTableName(),
constraintViolated);
}
List<RSGroupInfo> rsGroupInfoList = RS_GROUP_ADMIN_CLIENT.listRSGroups();
List<RSGroupInfo> rsGroupInfoList = ADMIN.listRSGroups();
boolean foundTable2 = false;
boolean foundTable1 = false;
for (int i = 0; i < rsGroupInfoList.size(); i++) {
if (rsGroupInfoList.get(i).getTables().contains(tableDescTwo.getTableName())) {
Set<TableName> tables =
Sets.newHashSet(ADMIN.listTablesInRSGroup(rsGroupInfoList.get(i).getName()));
if (tables.contains(tableDescTwo.getTableName())) {
foundTable2 = true;
}
if (rsGroupInfoList.get(i).getTables().contains(tableDescOne.getTableName())) {
if (tables.contains(tableDescOne.getTableName())) {
foundTable1 = true;
}
}
@ -458,11 +459,10 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase {
TEST_UTIL.waitFor(5000, new Waiter.Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
return (MASTER.getMasterProcedureExecutor().getActiveExecutorCount() == 0);
return MASTER.getMasterProcedureExecutor().getActiveExecutorCount() == 0;
}
});
SortedSet<TableName> tables =
RS_GROUP_ADMIN_CLIENT.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables();
Set<TableName> tables = Sets.newHashSet(ADMIN.listTablesInRSGroup(RSGroupInfo.DEFAULT_GROUP));
assertTrue("Table 't1' must be in 'default' rsgroup", tables.contains(tn1));
// Cleanup

18
hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java
View File

@ -326,8 +326,9 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
LOG.debug("Print group info : " + ADMIN.listRSGroups());
int oldDefaultGroupServerSize = ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().size();
int oldDefaultGroupTableSize =
RS_GROUP_ADMIN_CLIENT.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size();
int oldDefaultGroupTableSize = ADMIN.listTablesInRSGroup(RSGroupInfo.DEFAULT_GROUP).size();
assertTrue(OBSERVER.preListTablesInRSGroupCalled);
assertTrue(OBSERVER.postListTablesInRSGroupCalled);
// test fail bogus server move
try {
@ -350,11 +351,14 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
assertEquals(oldDefaultGroupServerSize,
ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().size());
assertEquals(oldDefaultGroupTableSize,
RS_GROUP_ADMIN_CLIENT.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size());
ADMIN.listTablesInRSGroup(RSGroupInfo.DEFAULT_GROUP).size());
// verify new group info
assertEquals(1, ADMIN.getRSGroup(newGroup.getName()).getServers().size());
assertEquals(0, RS_GROUP_ADMIN_CLIENT.getRSGroupInfo(newGroup.getName()).getTables().size());
assertEquals(0,
ADMIN.getConfiguredNamespacesAndTablesInRSGroup(newGroup.getName()).getSecond().size());
assertTrue(OBSERVER.preGetConfiguredNamespacesAndTablesInRSGroupCalled);
assertTrue(OBSERVER.postGetConfiguredNamespacesAndTablesInRSGroupCalled);
// get all region to move targetServer
List<String> regionList = getTableRegionMap().get(tableName);
@ -396,12 +400,12 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
// verify tables' not exist in old group
Set<TableName> defaultTables =
RS_GROUP_ADMIN_CLIENT.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables();
Sets.newHashSet(ADMIN.listTablesInRSGroup(RSGroupInfo.DEFAULT_GROUP));
assertFalse(defaultTables.contains(tableName));
// verify tables' exist in new group
Set<TableName> newGroupTables =
RS_GROUP_ADMIN_CLIENT.getRSGroupInfo(newGroup.getName()).getTables();
Set<TableName> newGroupTables = Sets
.newHashSet(ADMIN.getConfiguredNamespacesAndTablesInRSGroup(newGroup.getName()).getSecond());
assertTrue(newGroupTables.contains(tableName));
// verify that all region still assign on targetServer

59
hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
View File

@ -31,6 +31,7 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ThreadLocalRandom;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterMetrics;
import org.apache.hadoop.hbase.ClusterMetrics.Option;
import org.apache.hadoop.hbase.HBaseCluster;
@ -76,7 +77,6 @@ public abstract class TestRSGroupsBase {
protected static HMaster MASTER;
protected boolean INIT = false;
protected static CPMasterObserver OBSERVER;
protected static RSGroupAdminClient RS_GROUP_ADMIN_CLIENT;
public final static long WAIT_TIMEOUT = 60000;
public final static int NUM_SLAVES_BASE = 4; // number of slaves for the smallest cluster
@ -92,14 +92,23 @@ public abstract class TestRSGroupsBase {
}
public static void setUpTestBeforeClass() throws Exception {
TEST_UTIL.getConfiguration().setFloat("hbase.master.balancer.stochastic.tableSkewCost", 6000);
TEST_UTIL.getConfiguration().setBoolean(RSGroupInfoManager.RS_GROUP_ENABLED, true);
TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
RSGroupAdminEndpoint.class.getName() + "," + CPMasterObserver.class.getName());
TEST_UTIL.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
Configuration conf = TEST_UTIL.getConfiguration();
conf.setFloat("hbase.master.balancer.stochastic.tableSkewCost", 6000);
if (conf.get(RSGroupInfoManager.RS_GROUP_ENABLED) == null) {
conf.setBoolean(RSGroupInfoManager.RS_GROUP_ENABLED, true);
}
if (conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) != null) {
conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) + "," +
CPMasterObserver.class.getName());
} else {
conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, CPMasterObserver.class.getName());
}
conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
NUM_SLAVES_BASE - 1);
TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
TEST_UTIL.getConfiguration().setInt("hbase.rpc.timeout", 100000);
conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
conf.setInt("hbase.rpc.timeout", 100000);
TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1);
initialize();
@ -121,7 +130,6 @@ public abstract class TestRSGroupsBase {
ADMIN.balancerSwitch(false, true);
MasterCoprocessorHost host = MASTER.getMasterCoprocessorHost();
OBSERVER = (CPMasterObserver) host.findCoprocessor(CPMasterObserver.class.getName());
RS_GROUP_ADMIN_CLIENT = new RSGroupAdminClient(TEST_UTIL.getConnection());
}
public static void tearDownAfterClass() throws Exception {
@ -319,6 +327,10 @@ public abstract class TestRSGroupsBase {
boolean postGetRSGroupInfoOfServerCalled = false;
boolean preSetRSGroupForTablesCalled = false;
boolean postSetRSGroupForTablesCalled = false;
boolean preListTablesInRSGroupCalled = false;
boolean postListTablesInRSGroupCalled = false;
boolean preGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
boolean postGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
public void resetFlags() {
preBalanceRSGroupCalled = false;
@ -345,6 +357,10 @@ public abstract class TestRSGroupsBase {
postGetRSGroupInfoOfServerCalled = false;
preSetRSGroupForTablesCalled = false;
postSetRSGroupForTablesCalled = false;
preListTablesInRSGroupCalled = false;
postListTablesInRSGroupCalled = false;
preGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
postGetConfiguredNamespacesAndTablesInRSGroupCalled = false;
}
@Override
@ -483,6 +499,29 @@ public abstract class TestRSGroupsBase {
final Address server) throws IOException {
postGetRSGroupInfoOfServerCalled = true;
}
}
@Override
public void preListTablesInRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx,
String groupName) throws IOException {
preListTablesInRSGroupCalled = true;
}
@Override
public void postListTablesInRSGroup(ObserverContext<MasterCoprocessorEnvironment> ctx,
String groupName) throws IOException {
postListTablesInRSGroupCalled = true;
}
@Override
public void preGetConfiguredNamespacesAndTablesInRSGroup(
ObserverContext<MasterCoprocessorEnvironment> ctx, String groupName) throws IOException {
preGetConfiguredNamespacesAndTablesInRSGroupCalled = true;
}
@Override
public void postGetConfiguredNamespacesAndTablesInRSGroup(
ObserverContext<MasterCoprocessorEnvironment> ctx, String groupName) throws IOException {
postGetConfiguredNamespacesAndTablesInRSGroupCalled = true;
}
}
}

5
hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java
View File

@ -164,7 +164,8 @@ public class TestRSGroupsKillRS extends TestRSGroupsBase {
Set<TableName> toAddTables = new HashSet<>();
toAddTables.add(tableName);
ADMIN.setRSGroup(toAddTables, groupName);
assertTrue(RS_GROUP_ADMIN_CLIENT.getRSGroupInfo(groupName).getTables().contains(tableName));
assertTrue(
ADMIN.getConfiguredNamespacesAndTablesInRSGroup(groupName).getSecond().contains(tableName));
TEST_UTIL.waitTableAvailable(tableName, 30000);
// check my_group servers and table regions
@ -241,7 +242,7 @@ public class TestRSGroupsKillRS extends TestRSGroupsBase {
Set<TableName> toAddTables = new HashSet<>();
toAddTables.add(TableName.META_TABLE_NAME);
ADMIN.setRSGroup(toAddTables, groupName);
assertTrue(RS_GROUP_ADMIN_CLIENT.getRSGroupInfo(groupName).getTables()
assertTrue(ADMIN.getConfiguredNamespacesAndTablesInRSGroup(groupName).getSecond()
.contains(TableName.META_TABLE_NAME));
// restart the regionserver in meta_group, and lower its version

12
hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java
View File

@ -78,6 +78,7 @@ import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
@ -779,6 +780,17 @@ public class VerifyingRSGroupAdmin implements Admin, Closeable {
return admin.listRSGroups();
}
@Override
public List<TableName> listTablesInRSGroup(String groupName) throws IOException {
return admin.listTablesInRSGroup(groupName);
}
@Override
public Pair<List<String>, List<TableName>>
getConfiguredNamespacesAndTablesInRSGroup(String groupName) throws IOException {
return admin.getConfiguredNamespacesAndTablesInRSGroup(groupName);
}
public void removeRSGroup(String groupName) throws IOException {
admin.removeRSGroup(groupName);
verify();

12
hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java
View File

@ -71,6 +71,7 @@ import org.apache.hadoop.hbase.thrift2.generated.TNamespaceDescriptor;
import org.apache.hadoop.hbase.thrift2.generated.TTableDescriptor;
import org.apache.hadoop.hbase.thrift2.generated.TTableName;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.thrift.TException;
import org.apache.thrift.transport.TTransport;
import org.apache.yetus.audience.InterfaceAudience;
@ -1226,4 +1227,15 @@ public class ThriftAdmin implements Admin {
public Future<Void> splitRegionAsync(byte[] regionName) throws IOException {
return splitRegionAsync(regionName, null);
}
@Override
public List<TableName> listTablesInRSGroup(String groupName) throws IOException {
throw new NotImplementedException("setRSGroup not supported in ThriftAdmin");
}
@Override
public Pair<List<String>, List<TableName>>
getConfiguredNamespacesAndTablesInRSGroup(String groupName) throws IOException {
throw new NotImplementedException("setRSGroup not supported in ThriftAdmin");
}
}