Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-23818 Cleanup the remaining RSGroupInfo.getTables call in the code base (#1152) 

Signed-off-by: stack <stack@apache.org>
Signed-off-by: Viraj Jasani <vjasani@apache.org>
This commit is contained in:
Duo Zhang 2020-02-12 09:45:14 +08:00 committed by Duo Zhang
parent 37e87aeca7
commit 7f2d823164
17 changed files with 71 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
hbase-client/src/main/java/org/apache/hadoop/hbase/RSGroupTableAccessor.java
View File

@ -1,86 +0,0 @@
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos;
import org.apache.hadoop.hbase.rsgroup.RSGroupInfo;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.yetus.audience.InterfaceAudience;
/**
* Read rs group information from <code>hbase:rsgroup</code>.
*/
@InterfaceAudience.Private
public final class RSGroupTableAccessor {
//Assigned before user tables
private static final TableName RSGROUP_TABLE_NAME =
TableName.valueOf(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR, "rsgroup");
private static final byte[] META_FAMILY_BYTES = Bytes.toBytes("m");
private static final byte[] META_QUALIFIER_BYTES = Bytes.toBytes("i");
private RSGroupTableAccessor() {
}
public static boolean isRSGroupsEnabled(Connection connection) throws IOException {
return connection.getAdmin().tableExists(RSGROUP_TABLE_NAME);
}
public static List<RSGroupInfo> getAllRSGroupInfo(Connection connection)
throws IOException {
try (Table rsGroupTable = connection.getTable(RSGROUP_TABLE_NAME)) {
List<RSGroupInfo> rsGroupInfos = new ArrayList<>();
for (Result result : rsGroupTable.getScanner(new Scan())) {
RSGroupInfo rsGroupInfo = getRSGroupInfo(result);
if (rsGroupInfo != null) {
rsGroupInfos.add(rsGroupInfo);
}
}
return rsGroupInfos;
}
}
private static RSGroupInfo getRSGroupInfo(Result result) throws IOException {
byte[] rsGroupInfo = result.getValue(META_FAMILY_BYTES, META_QUALIFIER_BYTES);
if (rsGroupInfo == null) {
return null;
}
RSGroupProtos.RSGroupInfo proto =
RSGroupProtos.RSGroupInfo.parseFrom(rsGroupInfo);
return ProtobufUtil.toGroupInfo(proto);
}
public static RSGroupInfo getRSGroupInfo(Connection connection, byte[] rsGroupName)
throws IOException {
try (Table rsGroupTable = connection.getTable(RSGROUP_TABLE_NAME)){
Result result = rsGroupTable.get(new Get(rsGroupName));
return getRSGroupInfo(result);
}
}
}

2
hbase-it/src/test/java/org/apache/hadoop/hbase/rsgroup/IntegrationTestRSGroup.java
View File

@ -46,7 +46,7 @@ public class IntegrationTestRSGroup extends TestRSGroupsBase {
TEST_UTIL = new IntegrationTestingUtility();
TEST_UTIL.getConfiguration().set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
RSGroupBasedLoadBalancer.class.getName());
TEST_UTIL.getConfiguration().setBoolean(RSGroupInfoManager.RS_GROUP_ENABLED, true);
RSGroupUtil.enableRSGroup(TEST_UTIL.getConfiguration());
((IntegrationTestingUtility) TEST_UTIL).initializeCluster(NUM_SLAVES_BASE);
// set shared configs
ADMIN = TEST_UTIL.getAdmin();

3
hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon
View File

@ -53,6 +53,7 @@ org.apache.hadoop.hbase.master.ServerManager;
org.apache.hadoop.hbase.protobuf.ProtobufUtil;
org.apache.hadoop.hbase.quotas.QuotaUtil;
org.apache.hadoop.hbase.rsgroup.RSGroupInfoManager;
org.apache.hadoop.hbase.rsgroup.RSGroupUtil;
org.apache.hadoop.hbase.security.access.PermissionStorage;
org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
@ -225,7 +226,7 @@ AssignmentManager assignmentManager = master.getAssignmentManager();
<& AssignmentManagerStatusTmpl; assignmentManager=master.getAssignmentManager()&>
</%if>
<%if !master.isInMaintenanceMode() %>
<%if master.getConfiguration().getBoolean(RSGroupInfoManager.RS_GROUP_ENABLED, false) &&
<%if RSGroupUtil.isRSGroupEnabled(master.getConfiguration()) &&
serverManager.getOnlineServersList().size() > 0 %>
<section>
<h2><a name="rsgroup">RSGroup</a></h2>

4
hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RSGroupListTmpl.jamon
View File

@ -33,10 +33,10 @@ ServerManager serverManager;
org.apache.hadoop.hbase.RegionMetrics;
org.apache.hadoop.hbase.ServerMetrics;
org.apache.hadoop.hbase.Size;
org.apache.hadoop.hbase.RSGroupTableAccessor;
org.apache.hadoop.hbase.master.ServerManager;
org.apache.hadoop.hbase.net.Address;
org.apache.hadoop.hbase.rsgroup.RSGroupInfo;
org.apache.hadoop.hbase.rsgroup.RSGroupUtil;
org.apache.hadoop.util.StringUtils;
org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
</%import>
@ -128,7 +128,7 @@ if (master.getServerManager() != null) {
deadServers++;
}
}
tables = rsGroupInfo.getTables().size();
tables = RSGroupUtil.listTablesInRSGroup(master, rsGroupInfo.getName()).size();
totalTables += tables;
double avgLoad = onlineServers == 0 ? 0 :
(double)numRegionsOnline / (double)onlineServers;

3
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
View File

@ -189,6 +189,7 @@ import org.apache.hadoop.hbase.replication.regionserver.ReplicationStatus;
import org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint;
import org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer;
import org.apache.hadoop.hbase.rsgroup.RSGroupInfoManager;
import org.apache.hadoop.hbase.rsgroup.RSGroupUtil;
import org.apache.hadoop.hbase.security.AccessDeniedException;
import org.apache.hadoop.hbase.security.SecurityConstants;
import org.apache.hadoop.hbase.security.UserProvider;
@ -807,7 +808,7 @@ public class HMaster extends HRegionServer implements MasterServices {
if (cpClasses != null) {
for (String cpClass : cpClasses) {
if (RSGroupAdminEndpoint.class.getName().equals(cpClass)) {
conf.setBoolean(RSGroupInfoManager.RS_GROUP_ENABLED, true);
RSGroupUtil.enableRSGroup(conf);
break;
}
}

10
hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
View File

@ -3177,14 +3177,8 @@ public class MasterRpcServices extends RSRpcServices implements
if (master.getMasterCoprocessorHost() != null) {
master.getMasterCoprocessorHost().preListTablesInRSGroup(groupName);
}
boolean isDefaultGroup = RSGroupInfo.DEFAULT_GROUP.equals(groupName);
for (TableDescriptor td : master.getTableDescriptors().getAll().values()) {
// no config means in default group
if (RSGroupUtil.getRSGroupInfo(master, master.getRSGroupInfoManager(), td.getTableName())
.map(g -> g.getName().equals(groupName)).orElse(isDefaultGroup)) {
builder.addTableName(ProtobufUtil.toProtoTableName(td.getTableName()));
}
}
RSGroupUtil.listTablesInRSGroup(master, groupName).stream()
.map(ProtobufUtil::toProtoTableName).forEach(builder::addTableName);
if (master.getMasterCoprocessorHost() != null) {
master.getMasterCoprocessorHost().postListTablesInRSGroup(groupName);
}

4
hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManager.java
View File

@ -32,8 +32,6 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public interface RSGroupInfoManager {
public static final String RS_GROUP_ENABLED = "hbase.balancer.rsgroup.enabled";
void start();
/**
@ -84,7 +82,7 @@ public interface RSGroupInfoManager {
RSGroupInfo getRSGroupForTable(TableName tableName) throws IOException;
static RSGroupInfoManager create(MasterServices master) throws IOException {
if (master.getConfiguration().getBoolean(RS_GROUP_ENABLED, false)) {
if (RSGroupUtil.isRSGroupEnabled(master.getConfiguration())) {
return RSGroupInfoManagerImpl.getInstance(master);
} else {
return new DisabledRSGroupInfoManager(master.getServerManager());

50
hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupMajorCompactionTTL.java
View File

@ -41,8 +41,8 @@ import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException;
/**
* This script takes an rsgroup as argument and compacts part/all of regions of that table
* based on the table's TTL.
* This script takes an rsgroup as argument and compacts part/all of regions of that table based on
* the table's TTL.
*/
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
public class RSGroupMajorCompactionTTL extends MajorCompactorTTL {
@ -55,24 +55,20 @@ public class RSGroupMajorCompactionTTL extends MajorCompactorTTL {
}
public int compactTTLRegionsOnGroup(Configuration conf, String rsgroup, int concurrency,
long sleep, int numServers, int numRegions, boolean dryRun, boolean skipWait)
throws Exception {
Connection conn = ConnectionFactory.createConnection(conf);
Admin admin = conn.getAdmin();
RSGroupInfo rsGroupInfo = admin.getRSGroup(rsgroup);
if (rsGroupInfo == null) {
LOG.error("Invalid rsgroup specified: " + rsgroup);
throw new IllegalArgumentException("Invalid rsgroup specified: " + rsgroup);
}
for (TableName tableName : rsGroupInfo.getTables()) {
int status = compactRegionsTTLOnTable(conf, tableName.getNameAsString(), concurrency, sleep,
long sleep, int numServers, int numRegions, boolean dryRun, boolean skipWait) throws Exception {
try (Connection conn = ConnectionFactory.createConnection(conf);
Admin admin = conn.getAdmin()) {
if (admin.getRSGroup(rsgroup) == null) {
LOG.error("Invalid rsgroup specified: " + rsgroup);
throw new IllegalArgumentException("Invalid rsgroup specified: " + rsgroup);
}
for (TableName tableName : admin.listTablesInRSGroup(rsgroup)) {
int status = compactRegionsTTLOnTable(conf, tableName.getNameAsString(), concurrency, sleep,
numServers, numRegions, dryRun, skipWait);
if (status != 0) {
LOG.error("Failed to compact table: " + tableName);
return status;
if (status != 0) {
LOG.error("Failed to compact table: " + tableName);
return status;
}
}
}
return 0;
@ -81,13 +77,8 @@ public class RSGroupMajorCompactionTTL extends MajorCompactorTTL {
protected Options getOptions() {
Options options = getCommonOptions();
options.addOption(
Option.builder("rsgroup")
.required()
.desc("Tables of rsgroup to be compacted")
.hasArg()
.build()
);
options.addOption(Option.builder("rsgroup").required().desc("Tables of rsgroup to be compacted")
.hasArg().build());
return options;
}
@ -101,9 +92,8 @@ public class RSGroupMajorCompactionTTL extends MajorCompactorTTL {
try {
commandLine = cmdLineParser.parse(options, args);
} catch (ParseException parseException) {
System.out.println(
"ERROR: Unable to parse command-line arguments " + Arrays.toString(args) + " due to: "
+ parseException);
System.out.println("ERROR: Unable to parse command-line arguments " + Arrays.toString(args) +
" due to: " + parseException);
printUsage(options);
return -1;
}
@ -123,7 +113,7 @@ public class RSGroupMajorCompactionTTL extends MajorCompactorTTL {
Configuration conf = getConf();
return compactTTLRegionsOnGroup(conf, rsgroup, concurrency, sleep, numServers, numRegions,
dryRun, skipWait);
dryRun, skipWait);
}
public static void main(String[] args) throws Exception {

27
hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupUtil.java
View File

@ -11,9 +11,12 @@
package org.apache.hadoop.hbase.rsgroup;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
import java.util.function.Predicate;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.TableDescriptor;
@ -31,9 +34,33 @@ public final class RSGroupUtil {
private static final Logger LOG = LoggerFactory.getLogger(RSGroupUtil.class);
public static final String RS_GROUP_ENABLED = "hbase.balancer.rsgroup.enabled";
private RSGroupUtil() {
}
public static boolean isRSGroupEnabled(Configuration conf) {
return conf.getBoolean(RS_GROUP_ENABLED, false);
}
public static void enableRSGroup(Configuration conf) {
conf.setBoolean(RS_GROUP_ENABLED, true);
}
public static List<TableName> listTablesInRSGroup(MasterServices master, String groupName)
throws IOException {
List<TableName> tables = new ArrayList<>();
boolean isDefaultGroup = RSGroupInfo.DEFAULT_GROUP.equals(groupName);
for (TableDescriptor td : master.getTableDescriptors().getAll().values()) {
// no config means in default group
if (RSGroupUtil.getRSGroupInfo(master, master.getRSGroupInfoManager(), td.getTableName())
.map(g -> g.getName().equals(groupName)).orElse(isDefaultGroup)) {
tables.add(td.getTableName());
}
}
return tables;
}
/**
* Will try to get the rsgroup from {@link TableDescriptor} first, and then try to get the rsgroup
* from the {@link NamespaceDescriptor}. If still not present, return empty.

10
hbase-server/src/main/resources/hbase-webapps/master/rsgroup.jsp
View File

@ -27,7 +27,6 @@
import="java.util.stream.Stream"
import="java.util.stream.Collectors"
import="org.apache.hadoop.hbase.HTableDescriptor"
import="org.apache.hadoop.hbase.RSGroupTableAccessor"
import="org.apache.hadoop.hbase.ServerName"
import="org.apache.hadoop.hbase.TableName"
import="org.apache.hadoop.hbase.client.Admin"
@ -38,6 +37,7 @@
import="org.apache.hadoop.hbase.master.RegionState"
import="org.apache.hadoop.hbase.net.Address"
import="org.apache.hadoop.hbase.rsgroup.RSGroupInfo"
import="org.apache.hadoop.hbase.rsgroup.RSGroupUtil"
import="org.apache.hadoop.hbase.util.Bytes"
import="org.apache.hadoop.hbase.util.VersionInfo"
import="org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix"%>
@ -57,7 +57,7 @@
RSGroupInfo rsGroupInfo = null;
final String ZEROKB = "0 KB";
final String ZEROMB = "0 MB";
if (!RSGroupTableAccessor.isRSGroupsEnabled(master.getConnection())) {
if (!RSGroupUtil.isRSGroupEnabled(master.getConfiguration())) {
%>
<div class="row inner_header">
<div class="page-header">
@ -67,8 +67,8 @@
<jsp:include page="redirect.jsp" />
<%
} else if (rsGroupName == null || rsGroupName.isEmpty() ||
(rsGroupInfo = RSGroupTableAccessor.getRSGroupInfo(
master.getConnection(), Bytes.toBytes(rsGroupName))) == null) {
(rsGroupInfo = master.getRSGroupInfoManager().getRSGroup(
rsGroupName)) == null) {
%>
<div class="row inner_header">
<div class="page-header">
@ -81,7 +81,7 @@
List<Address> rsGroupServers = new ArrayList<>();
List<TableName> rsGroupTables = new ArrayList<>();
rsGroupServers.addAll(rsGroupInfo.getServers());
rsGroupTables.addAll(rsGroupInfo.getTables());
rsGroupTables.addAll(RSGroupUtil.listTablesInRSGroup(master, rsGroupInfo.getName()));
Collections.sort(rsGroupServers);
rsGroupTables.sort((o1, o2) -> {
int compare = Bytes.compareTo(o1.getNamespace(), o2.getNamespace());

2
hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestEnableRSGroups.java
View File

@ -33,7 +33,7 @@ public class TestEnableRSGroups extends EnableRSGroupsTestBase {
@Override
protected void enableRSGroup(Configuration conf) {
conf.setBoolean(RSGroupInfoManager.RS_GROUP_ENABLED, true);
conf.setBoolean(RSGroupUtil.RS_GROUP_ENABLED, true);
}
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestMigrateRSGroupInfo.java
View File

@ -72,7 +72,7 @@ public class TestMigrateRSGroupInfo extends TestRSGroupsBase {
TEST_UTIL.getConfiguration().setClass(HConstants.MASTER_IMPL, HMasterForTest.class,
HMaster.class);
// confirm that we could enable rs group by setting the old CP.
TEST_UTIL.getConfiguration().setBoolean(RSGroupInfoManager.RS_GROUP_ENABLED, false);
TEST_UTIL.getConfiguration().setBoolean(RSGroupUtil.RS_GROUP_ENABLED, false);
TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
RSGroupAdminEndpoint.class.getName());
setUpTestBeforeClass();

2
hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupMajorCompactionTTL.java
View File

@ -53,7 +53,7 @@ public class TestRSGroupMajorCompactionTTL extends TestMajorCompactorTTL {
public void setUp() throws Exception {
utility = new HBaseTestingUtility();
Configuration conf = utility.getConfiguration();
conf.setBoolean(RSGroupInfoManager.RS_GROUP_ENABLED, true);
RSGroupUtil.enableRSGroup(conf);
conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_SLAVES_BASE);
conf.setInt("hbase.hfile.compaction.discharger.interval", 10);
utility.startMiniCluster(NUM_SLAVES_BASE);

4
hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
View File

@ -94,8 +94,8 @@ public abstract class TestRSGroupsBase {
public static void setUpTestBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
conf.setFloat("hbase.master.balancer.stochastic.tableSkewCost", 6000);
if (conf.get(RSGroupInfoManager.RS_GROUP_ENABLED) == null) {
conf.setBoolean(RSGroupInfoManager.RS_GROUP_ENABLED, true);
if (conf.get(RSGroupUtil.RS_GROUP_ENABLED) == null) {
RSGroupUtil.enableRSGroup(conf);
}
if (conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) != null) {
conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,

2
hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java
View File

@ -77,7 +77,7 @@ public class TestRSGroupsOfflineMode extends TestRSGroupsBase {
@BeforeClass
public static void setUp() throws Exception {
TEST_UTIL = new HBaseTestingUtility();
TEST_UTIL.getConfiguration().setBoolean(RSGroupInfoManager.RS_GROUP_ENABLED, true);
RSGroupUtil.enableRSGroup(TEST_UTIL.getConfiguration());
TEST_UTIL.getConfiguration().set(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, "1");
StartMiniClusterOption option =
StartMiniClusterOption.builder().numMasters(2).numRegionServers(3).numDataNodes(3).build();

2
hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsWithACL.java
View File

@ -107,7 +107,7 @@ public class TestRSGroupsWithACL extends SecureTestUtil {
// Verify enableSecurity sets up what we require
verifyConfiguration(conf);
// Enable rsgroup
conf.setBoolean(RSGroupInfoManager.RS_GROUP_ENABLED, true);
RSGroupUtil.enableRSGroup(conf);
TEST_UTIL.startMiniCluster();
// Wait for the ACL table to become available

4
hbase-shell/src/test/java/org/apache/hadoop/hbase/client/TestRSGroupShell.java
View File

@ -19,7 +19,7 @@ package org.apache.hadoop.hbase.client;
import java.io.IOException;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.rsgroup.RSGroupInfoManager;
import org.apache.hadoop.hbase.rsgroup.RSGroupUtil;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.jruby.embed.PathType;
@ -40,7 +40,7 @@ public class TestRSGroupShell extends AbstractTestShell {
setUpConfig();
// enable rs group
TEST_UTIL.getConfiguration().setBoolean(RSGroupInfoManager.RS_GROUP_ENABLED, true);
RSGroupUtil.enableRSGroup(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);