HBASE-25492: Create table with RSGroup (#2883)
Added new interface in TableDescriptor which allows user to define RSGroup name while creating or modifying a table. Signed-off-by: Duo Zhang <zhangduo@apache.org> Signed-off-by: Pankaj Kumar<pankajkumar@apache.org>
This commit is contained in:
Mohammad Arshad
GitHub
parent
c9d0ec903d
commit
0487760829
|
|
@ -22,6 +22,7 @@ import java.io.IOException;
|
|||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
|
@ -987,4 +988,9 @@ public class HTableDescriptor implements TableDescriptor, Comparable<HTableDescr
|
|||
protected ModifyableTableDescriptor getDelegateeForModification() {
|
||||
return delegatee;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<String> getRegionServerGroup() {
|
||||
return delegatee.getRegionServerGroup();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import java.util.Collection;
|
|||
import java.util.Comparator;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
|
@ -360,4 +361,11 @@ public interface TableDescriptor {
|
|||
}
|
||||
return !enabled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the region server group this table belongs to. The regions of this table will be placed
|
||||
* only on the region servers within this group. If not present, will be placed on
|
||||
* {@link org.apache.hadoop.hbase.rsgroup.RSGroupInfo#DEFAULT_GROUP}.
|
||||
*/
|
||||
Optional<String> getRegionServerGroup();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.HConstants;
|
|||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.exceptions.DeserializationException;
|
||||
import org.apache.hadoop.hbase.exceptions.HBaseException;
|
||||
import org.apache.hadoop.hbase.rsgroup.RSGroupInfo;
|
||||
import org.apache.hadoop.hbase.security.User;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.PrettyPrinter;
|
||||
|
|
@ -192,6 +193,9 @@ public class TableDescriptorBuilder {
|
|||
private static final Bytes PRIORITY_KEY
|
||||
= new Bytes(Bytes.toBytes(PRIORITY));
|
||||
|
||||
private static final Bytes RSGROUP_KEY =
|
||||
new Bytes(Bytes.toBytes(RSGroupInfo.TABLE_DESC_PROP_GROUP));
|
||||
|
||||
/**
|
||||
* Relative priority of the table used for rpc scheduling
|
||||
*/
|
||||
|
|
@ -594,6 +598,17 @@ public class TableDescriptorBuilder {
|
|||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the RSGroup for this table, specified RSGroup must exist before create or modify table.
|
||||
*
|
||||
* @param group rsgroup name
|
||||
* @return a TableDescriptorBuilder
|
||||
*/
|
||||
public TableDescriptorBuilder setRegionServerGroup(String group) {
|
||||
desc.setValue(RSGROUP_KEY, group);
|
||||
return this;
|
||||
}
|
||||
|
||||
public TableDescriptor build() {
|
||||
return new ModifyableTableDescriptor(desc);
|
||||
}
|
||||
|
|
@ -1651,6 +1666,16 @@ public class TableDescriptorBuilder {
|
|||
public int getColumnFamilyCount() {
|
||||
return families.size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<String> getRegionServerGroup() {
|
||||
Bytes value = values.get(RSGROUP_KEY);
|
||||
if (value != null) {
|
||||
return Optional.of(Bytes.toString(value.get(), value.getOffset(), value.getLength()));
|
||||
} else {
|
||||
return Optional.empty();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static Optional<CoprocessorDescriptor> toCoprocessorDescriptor(String spec) {
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ import org.apache.yetus.audience.InterfaceAudience;
|
|||
public class RSGroupInfo {
|
||||
public static final String DEFAULT_GROUP = "default";
|
||||
public static final String NAMESPACE_DESC_PROP_GROUP = "hbase.rsgroup.name";
|
||||
public static final String TABLE_DESC_PROP_GROUP = "hbase.rsgroup.name";
|
||||
|
||||
private final String name;
|
||||
// Keep servers in a sorted set so has an expected ordering when displayed.
|
||||
|
|
|
|||
|
|
@ -485,7 +485,7 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
|
|||
}
|
||||
|
||||
void assignTableToGroup(TableDescriptor desc) throws IOException {
|
||||
RSGroupInfo rsGroupInfo = groupInfoManager.determineRSGroupInfoForTable(desc.getTableName());
|
||||
RSGroupInfo rsGroupInfo = determineRSGroupInfoForTable(desc);
|
||||
if (rsGroupInfo == null) {
|
||||
throw new ConstraintException("Default RSGroup for this table " + desc.getTableName()
|
||||
+ " does not exist.");
|
||||
|
|
@ -508,21 +508,75 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
|
|||
if (desc.getTableName().isSystemTable()) {
|
||||
return;
|
||||
}
|
||||
RSGroupInfo rsGroupInfo = groupInfoManager.determineRSGroupInfoForTable(desc.getTableName());
|
||||
if (rsGroupInfo == null) {
|
||||
throw new ConstraintException("Default RSGroup for this table " + desc.getTableName()
|
||||
+ " does not exist.");
|
||||
}
|
||||
if (!RSGroupUtil.rsGroupHasOnlineServer(master, rsGroupInfo)) {
|
||||
throw new HBaseIOException("No online servers in the rsgroup " + rsGroupInfo.getName()
|
||||
+ " which table " + desc.getTableName().getNameAsString() + " belongs to");
|
||||
}
|
||||
synchronized (groupInfoManager) {
|
||||
groupInfoManager.moveTables(
|
||||
Collections.singleton(desc.getTableName()), rsGroupInfo.getName());
|
||||
moveTableToValidRSGroup(desc);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void preModifyTableAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
|
||||
TableName tableName, TableDescriptor currentDescriptor, TableDescriptor newDescriptor)
|
||||
throws IOException {
|
||||
// If table's rsgroup is changed, it must be valid
|
||||
if (!currentDescriptor.getRegionServerGroup().equals(newDescriptor.getRegionServerGroup())) {
|
||||
RSGroupInfo rsGroupInfo = determineRSGroupInfoForTable(newDescriptor);
|
||||
validateRSGroup(newDescriptor, rsGroupInfo);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void postCompletedModifyTableAction(ObserverContext<MasterCoprocessorEnvironment> ctx,
|
||||
TableName tableName, TableDescriptor oldDescriptor, TableDescriptor currentDescriptor)
|
||||
throws IOException {
|
||||
// If table's rsgroup is changed, move table into the rsgroup.
|
||||
if (!oldDescriptor.getRegionServerGroup().equals(currentDescriptor.getRegionServerGroup())) {
|
||||
RSGroupInfo rsGroupInfo = determineRSGroupInfoForTable(currentDescriptor);
|
||||
moveTableToRSGroup(currentDescriptor, rsGroupInfo);
|
||||
}
|
||||
}
|
||||
|
||||
// Determine and validate rs group then move table to this valid rs group.
|
||||
private void moveTableToValidRSGroup(TableDescriptor desc) throws IOException {
|
||||
RSGroupInfo rsGroupInfo = determineRSGroupInfoForTable(desc);
|
||||
validateRSGroup(desc, rsGroupInfo);
|
||||
moveTableToRSGroup(desc, rsGroupInfo);
|
||||
}
|
||||
|
||||
private void validateRSGroup(TableDescriptor desc, RSGroupInfo rsGroupInfo) throws IOException {
|
||||
if (rsGroupInfo == null) {
|
||||
throw new ConstraintException(
|
||||
"Default RSGroup for this table " + desc.getTableName() + " does not exist.");
|
||||
}
|
||||
if (!RSGroupUtil.rsGroupHasOnlineServer(master, rsGroupInfo)) {
|
||||
throw new HBaseIOException(
|
||||
"No online servers in the rsgroup " + rsGroupInfo.getName() + " which table " + desc
|
||||
.getTableName().getNameAsString() + " belongs to");
|
||||
}
|
||||
}
|
||||
|
||||
private void moveTableToRSGroup(final TableDescriptor desc, RSGroupInfo rsGroupInfo)
|
||||
throws IOException {
|
||||
// In case of modify table, when rs group is not changed, move is not required.
|
||||
if (!rsGroupInfo.containsTable(desc.getTableName())) {
|
||||
synchronized (groupInfoManager) {
|
||||
groupInfoManager
|
||||
.moveTables(Collections.singleton(desc.getTableName()), rsGroupInfo.getName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private RSGroupInfo determineRSGroupInfoForTable(final TableDescriptor desc) throws IOException {
|
||||
Optional<String> optGroupNameOfTable = desc.getRegionServerGroup();
|
||||
if (optGroupNameOfTable.isPresent()) {
|
||||
final RSGroupInfo rsGroup = groupInfoManager.getRSGroup(optGroupNameOfTable.get());
|
||||
if (rsGroup == null) {
|
||||
// When rs group is set in table descriptor then it must exist
|
||||
throw new ConstraintException(
|
||||
"Region server group " + optGroupNameOfTable.get() + " does not exist.");
|
||||
} else {
|
||||
return rsGroup;
|
||||
}
|
||||
}
|
||||
return groupInfoManager.determineRSGroupInfoForTable(desc.getTableName());
|
||||
}
|
||||
// Remove table from its RSGroup.
|
||||
@Override
|
||||
public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
|
||||
|
|
|
|||
|
|
@ -27,11 +27,15 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.stream.Collectors;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.NamespaceDescriptor;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.client.RegionInfo;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptor;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.client.TableState;
|
||||
import org.apache.hadoop.hbase.constraint.ConstraintException;
|
||||
import org.apache.hadoop.hbase.master.HMaster;
|
||||
|
|
@ -43,7 +47,9 @@ import org.apache.hadoop.hbase.master.ServerManager;
|
|||
import org.apache.hadoop.hbase.master.TableStateManager;
|
||||
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
|
||||
import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
|
||||
import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
|
||||
import org.apache.hadoop.hbase.net.Address;
|
||||
import org.apache.hadoop.hbase.procedure2.Procedure;
|
||||
import org.apache.hadoop.hbase.util.Pair;
|
||||
import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
|
|
@ -459,7 +465,7 @@ public class RSGroupAdminServer implements RSGroupAdmin {
|
|||
// targetGroup is null when a table is being deleted. In this case no further
|
||||
// action is required.
|
||||
if (targetGroup != null) {
|
||||
moveTableRegionsToGroup(tables, rsGroupInfoManager.getRSGroup(targetGroup));
|
||||
modifyOrMoveTables(tables, rsGroupInfoManager.getRSGroup(targetGroup));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -582,7 +588,7 @@ public class RSGroupAdminServer implements RSGroupAdmin {
|
|||
rsGroupInfoManager.getRSGroup(srcGroup).getServers(),
|
||||
targetGroup, srcGroup);
|
||||
//move regions of these tables which are not on group servers
|
||||
moveTableRegionsToGroup(tables, rsGroupInfoManager.getRSGroup(targetGroup));
|
||||
modifyOrMoveTables(tables, rsGroupInfoManager.getRSGroup(targetGroup));
|
||||
}
|
||||
LOG.info("Move servers and tables done. Severs: {}, Tables: {} => {}", servers, tables,
|
||||
targetGroup);
|
||||
|
|
@ -609,6 +615,11 @@ public class RSGroupAdminServer implements RSGroupAdmin {
|
|||
public void renameRSGroup(String oldName, String newName) throws IOException {
|
||||
synchronized (rsGroupInfoManager) {
|
||||
rsGroupInfoManager.renameRSGroup(oldName, newName);
|
||||
Set<TableDescriptor> updateTables = master.getTableDescriptors().getAll().values().stream()
|
||||
.filter(t -> oldName.equals(t.getRegionServerGroup().orElse(null)))
|
||||
.collect(Collectors.toSet());
|
||||
// Update rs group info into table descriptors
|
||||
modifyTablesAndWaitForCompletion(updateTables, newName);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -718,4 +729,65 @@ public class RSGroupAdminServer implements RSGroupAdmin {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Modify table or move table's regions
|
||||
void modifyOrMoveTables(Set<TableName> tables, RSGroupInfo targetGroup) throws IOException {
|
||||
Set<TableName> tablesToBeMoved = new HashSet<>(tables.size());
|
||||
Set<TableDescriptor> tablesToBeModified = new HashSet<>(tables.size());
|
||||
// Segregate tables into to be modified or to be moved category
|
||||
for (TableName tableName : tables) {
|
||||
TableDescriptor descriptor = master.getTableDescriptors().get(tableName);
|
||||
if (descriptor == null) {
|
||||
LOG.error(
|
||||
"TableDescriptor of table {} not found. Skipping the region movement of this table.");
|
||||
continue;
|
||||
}
|
||||
if (descriptor.getRegionServerGroup().isPresent()) {
|
||||
tablesToBeModified.add(descriptor);
|
||||
} else {
|
||||
tablesToBeMoved.add(tableName);
|
||||
}
|
||||
}
|
||||
List<Long> procedureIds = null;
|
||||
if (!tablesToBeModified.isEmpty()) {
|
||||
procedureIds = modifyTables(tablesToBeModified, targetGroup.getName());
|
||||
}
|
||||
if (!tablesToBeMoved.isEmpty()) {
|
||||
moveTableRegionsToGroup(tablesToBeMoved, targetGroup);
|
||||
}
|
||||
// By this time moveTableRegionsToGroup is finished, lets wait for modifyTables completion
|
||||
if (procedureIds != null) {
|
||||
waitForProcedureCompletion(procedureIds);
|
||||
}
|
||||
}
|
||||
|
||||
private void modifyTablesAndWaitForCompletion(Set<TableDescriptor> tableDescriptors,
|
||||
String targetGroup) throws IOException {
|
||||
final List<Long> procIds = modifyTables(tableDescriptors, targetGroup);
|
||||
waitForProcedureCompletion(procIds);
|
||||
}
|
||||
|
||||
// Modify table internally moves the regions as well. So separate region movement is not needed
|
||||
private List<Long> modifyTables(Set<TableDescriptor> tableDescriptors, String targetGroup)
|
||||
throws IOException {
|
||||
List<Long> procIds = new ArrayList<>(tableDescriptors.size());
|
||||
for (TableDescriptor oldTd : tableDescriptors) {
|
||||
TableDescriptor newTd =
|
||||
TableDescriptorBuilder.newBuilder(oldTd).setRegionServerGroup(targetGroup).build();
|
||||
procIds.add(master
|
||||
.modifyTable(oldTd.getTableName(), newTd, HConstants.NO_NONCE, HConstants.NO_NONCE));
|
||||
}
|
||||
return procIds;
|
||||
}
|
||||
|
||||
private void waitForProcedureCompletion(List<Long> procIds) throws IOException {
|
||||
for (long procId : procIds) {
|
||||
Procedure<?> proc = master.getMasterProcedureExecutor().getProcedure(procId);
|
||||
if (proc == null) {
|
||||
continue;
|
||||
}
|
||||
ProcedureSyncWait
|
||||
.waitForProcedureToCompleteIOE(master.getMasterProcedureExecutor(), proc, Long.MAX_VALUE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,413 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.hbase.rsgroup;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.regex.Pattern;
|
||||
import org.apache.hadoop.hbase.DoNotRetryIOException;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.Waiter;
|
||||
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
|
||||
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.client.SnapshotDescription;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptor;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.constraint.ConstraintException;
|
||||
import org.apache.hadoop.hbase.testclassification.LargeTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
|
||||
|
||||
@Category({ LargeTests.class })
|
||||
public class TestTableDescriptorWithRSGroup extends TestRSGroupsBase {
|
||||
|
||||
@ClassRule
|
||||
public static final HBaseClassTestRule CLASS_RULE =
|
||||
HBaseClassTestRule.forClass(TestTableDescriptorWithRSGroup.class);
|
||||
private final byte[] familyNameBytes = Bytes.toBytes("f1");
|
||||
|
||||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
setUpTestBeforeClass();
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDown() throws Exception {
|
||||
tearDownAfterClass();
|
||||
}
|
||||
|
||||
@Before
|
||||
public void beforeMethod() throws Exception {
|
||||
setUpBeforeMethod();
|
||||
}
|
||||
|
||||
@After
|
||||
public void afterMethod() throws Exception {
|
||||
tearDownAfterMethod();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateTableInTableDescriptorSpecificRSGroup() throws Exception {
|
||||
final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
|
||||
assertEquals(0, newGroup.getTables().size());
|
||||
|
||||
// assertion is done in createTableInRSGroup
|
||||
createTableWithRSGroupDetail(newGroup.getName());
|
||||
|
||||
// Create table should fail if specified rs group does not exist.
|
||||
try {
|
||||
TableDescriptor desc =
|
||||
TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName.getNameAsString() + "_2"))
|
||||
.setRegionServerGroup("nonExistingRSGroup")
|
||||
.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("f1")).build())
|
||||
.build();
|
||||
admin.createTable(desc, getSpitKeys(6));
|
||||
fail("Should have thrown ConstraintException but no exception thrown.");
|
||||
} catch (ConstraintException e) {
|
||||
assertEquals(e.getMessage(), "Region server group nonExistingRSGroup does not exist.");
|
||||
}
|
||||
}
|
||||
|
||||
private void createTableWithRSGroupDetail(String newGroup) throws Exception {
|
||||
// Create table
|
||||
|
||||
ColumnFamilyDescriptor f1 = ColumnFamilyDescriptorBuilder.newBuilder(familyNameBytes).build();
|
||||
TableDescriptor desc =
|
||||
TableDescriptorBuilder.newBuilder(tableName).setRegionServerGroup(newGroup)
|
||||
.setColumnFamily(f1).build();
|
||||
admin.createTable(desc, getSpitKeys(5));
|
||||
|
||||
TEST_UTIL.waitFor(WAIT_TIMEOUT, (Waiter.Predicate<Exception>) () -> {
|
||||
List<String> regions = getTableRegionMap().get(tableName);
|
||||
if (regions == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return getTableRegionMap().get(tableName).size() >= 5;
|
||||
});
|
||||
TableDescriptor descriptor = admin.getConnection().getTable(tableName).getDescriptor();
|
||||
Optional<String> regionServerGroup = descriptor.getRegionServerGroup();
|
||||
assertTrue("RSGroup info is not updated into TableDescriptor when table is created.",
|
||||
regionServerGroup.isPresent());
|
||||
assertEquals(newGroup, regionServerGroup.get());
|
||||
}
|
||||
|
||||
// moveTables should update rs group info in table descriptor
|
||||
@Test
|
||||
public void testMoveTablesShouldUpdateTableDescriptor() throws Exception {
|
||||
RSGroupInfo rsGroup1 = addGroup("rsGroup1", 1);
|
||||
assertEquals(0, rsGroup1.getTables().size());
|
||||
|
||||
createTableWithRSGroupDetail(rsGroup1.getName());
|
||||
TableDescriptor descriptor = admin.getConnection().getTable(tableName).getDescriptor();
|
||||
Optional<String> regionServerGroup = descriptor.getRegionServerGroup();
|
||||
assertTrue("RSGroup info is not updated into TableDescriptor when table created",
|
||||
regionServerGroup.isPresent());
|
||||
assertEquals(rsGroup1.getName(), regionServerGroup.get());
|
||||
|
||||
// moveTables
|
||||
RSGroupInfo rsGroup2 = addGroup("rsGroup2", 1);
|
||||
rsGroupAdmin.moveTables(Sets.newHashSet(tableName), rsGroup2.getName());
|
||||
descriptor = admin.getConnection().getTable(tableName).getDescriptor();
|
||||
regionServerGroup = descriptor.getRegionServerGroup();
|
||||
assertTrue("RSGroup info is not updated into TableDescriptor when table moved",
|
||||
regionServerGroup.isPresent());
|
||||
assertEquals(rsGroup2.getName(), regionServerGroup.get());
|
||||
}
|
||||
|
||||
// moveServersAndTables should update rs group info in table descriptor
|
||||
@Test
|
||||
public void testMoveServersAndTablesShouldUpdateTableDescriptor() throws Exception {
|
||||
RSGroupInfo rsGroup1 = addGroup("rsGroup1", 2);
|
||||
assertEquals(0, rsGroup1.getTables().size());
|
||||
|
||||
createTableWithRSGroupDetail(rsGroup1.getName());
|
||||
TableDescriptor descriptor = admin.getConnection().getTable(tableName).getDescriptor();
|
||||
Optional<String> regionServerGroup = descriptor.getRegionServerGroup();
|
||||
assertTrue("RSGroup info is not updated into TableDescriptor when table created",
|
||||
regionServerGroup.isPresent());
|
||||
assertEquals(rsGroup1.getName(), regionServerGroup.get());
|
||||
|
||||
// moveServersAndTables
|
||||
rsGroupAdmin.moveServersAndTables(Sets.newHashSet(rsGroup1.getServers().iterator().next()),
|
||||
Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
|
||||
|
||||
descriptor = admin.getConnection().getTable(tableName).getDescriptor();
|
||||
regionServerGroup = descriptor.getRegionServerGroup();
|
||||
assertTrue("RSGroup info is not updated into TableDescriptor when table moved",
|
||||
regionServerGroup.isPresent());
|
||||
assertEquals(RSGroupInfo.DEFAULT_GROUP, regionServerGroup.get());
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRenameRSGroupUpdatesTableDescriptor() throws Exception {
|
||||
RSGroupInfo oldGroup = addGroup("oldGroup", 1);
|
||||
assertEquals(0, oldGroup.getTables().size());
|
||||
|
||||
createTableWithRSGroupDetail(oldGroup.getName());
|
||||
|
||||
final String newGroupName = "newGroup";
|
||||
rsGroupAdmin.renameRSGroup(oldGroup.getName(), newGroupName);
|
||||
TableDescriptor descriptor = admin.getConnection().getTable(tableName).getDescriptor();
|
||||
Optional<String> regionServerGroup = descriptor.getRegionServerGroup();
|
||||
assertTrue("RSGroup info is not updated into TableDescriptor when rs group renamed",
|
||||
regionServerGroup.isPresent());
|
||||
assertEquals(newGroupName, regionServerGroup.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCloneSnapshotWithRSGroup() throws Exception {
|
||||
RSGroupInfo rsGroup1 = addGroup("rsGroup1", 1);
|
||||
assertEquals(0, rsGroup1.getTables().size());
|
||||
// Creates table in rsGroup1
|
||||
createTableWithRSGroupDetail(rsGroup1.getName());
|
||||
|
||||
// Create snapshot
|
||||
final String snapshotName = "snapShot2";
|
||||
admin.snapshot(snapshotName, tableName);
|
||||
final List<SnapshotDescription> snapshotDescriptions =
|
||||
admin.listSnapshots(Pattern.compile("snapShot2"));
|
||||
assertEquals(1, snapshotDescriptions.size());
|
||||
assertEquals(snapshotName, snapshotDescriptions.get(0).getName());
|
||||
|
||||
// Move table to different rs group then delete the old rs group
|
||||
RSGroupInfo rsGroup2 = addGroup("rsGroup2", 1);
|
||||
rsGroupAdmin.moveTables(Sets.newHashSet(tableName), rsGroup2.getName());
|
||||
rsGroup2 = rsGroupAdmin.getRSGroupInfo(rsGroup2.getName());
|
||||
assertEquals(1, rsGroup2.getTables().size());
|
||||
assertEquals(rsGroup2.getTables().first(), tableName);
|
||||
|
||||
// Clone Snapshot
|
||||
final TableName clonedTable1 = TableName.valueOf(tableName.getNameAsString() + "_1");
|
||||
admin.cloneSnapshot(Bytes.toBytes(snapshotName), clonedTable1);
|
||||
|
||||
// Verify that cloned table is created into old rs group
|
||||
final RSGroupInfo rsGroupInfoOfTable = rsGroupAdmin.getRSGroupInfoOfTable(clonedTable1);
|
||||
assertEquals(rsGroup1.getName(), rsGroupInfoOfTable.getName());
|
||||
TableDescriptor descriptor = admin.getConnection().getTable(clonedTable1).getDescriptor();
|
||||
Optional<String> regionServerGroup = descriptor.getRegionServerGroup();
|
||||
assertTrue("RSGroup info is not updated into TableDescriptor when table is cloned.",
|
||||
regionServerGroup.isPresent());
|
||||
assertEquals(rsGroup1.getName(), regionServerGroup.get());
|
||||
|
||||
// Delete table's original rs group, clone should fail.
|
||||
rsGroupAdmin
|
||||
.moveServersAndTables(Sets.newHashSet(rsGroup1.getServers()), Sets.newHashSet(clonedTable1),
|
||||
rsGroup2.getName());
|
||||
rsGroupAdmin.removeRSGroup(rsGroup1.getName());
|
||||
// Clone Snapshot
|
||||
final TableName clonedTable2 = TableName.valueOf(tableName.getNameAsString() + "_2");
|
||||
try {
|
||||
admin.cloneSnapshot(Bytes.toBytes(snapshotName), clonedTable2);
|
||||
fail("Should have thrown ConstraintException but no exception thrown.");
|
||||
} catch (ConstraintException e) {
|
||||
assertTrue(
|
||||
e.getCause().getMessage().contains("Region server group rsGroup1 does not exist."));
|
||||
}
|
||||
}
|
||||
|
||||
// Modify table should validate rs group existence and move rs group if required
|
||||
@Test
|
||||
public void testMoveTablesWhenModifyTableWithNewRSGroup() throws Exception {
|
||||
RSGroupInfo rsGroup1 = addGroup("rsGroup1", 1);
|
||||
assertEquals(0, rsGroup1.getTables().size());
|
||||
|
||||
createTableWithRSGroupDetail(rsGroup1.getName());
|
||||
TableDescriptor descriptor = admin.getConnection().getTable(tableName).getDescriptor();
|
||||
Optional<String> regionServerGroup = descriptor.getRegionServerGroup();
|
||||
assertTrue("RSGroup info is not updated into TableDescriptor when table created",
|
||||
regionServerGroup.isPresent());
|
||||
assertEquals(rsGroup1.getName(), regionServerGroup.get());
|
||||
|
||||
final TableDescriptor newTableDescriptor =
|
||||
TableDescriptorBuilder.newBuilder(descriptor).setRegionServerGroup("rsGroup2").build();
|
||||
|
||||
// ConstraintException as rsGroup2 does not exits
|
||||
try {
|
||||
admin.modifyTable(newTableDescriptor);
|
||||
fail("Should have thrown ConstraintException but no exception thrown.");
|
||||
} catch (ConstraintException e) {
|
||||
assertTrue(
|
||||
e.getCause().getMessage().contains("Region server group rsGroup2 does not exist."));
|
||||
}
|
||||
|
||||
addGroup("rsGroup2", 1);
|
||||
// Table creation should be successful as rsGroup2 exists.
|
||||
admin.modifyTable(newTableDescriptor);
|
||||
|
||||
// old group should not have table mapping now
|
||||
rsGroup1 = rsGroupAdmin.getRSGroupInfo("rsGroup1");
|
||||
assertEquals(0, rsGroup1.getTables().size());
|
||||
|
||||
RSGroupInfo rsGroup2 = rsGroupAdmin.getRSGroupInfo("rsGroup2");
|
||||
assertEquals(1, rsGroup2.getTables().size());
|
||||
descriptor = admin.getConnection().getTable(tableName).getDescriptor();
|
||||
regionServerGroup = descriptor.getRegionServerGroup();
|
||||
assertTrue("RSGroup info is not updated into TableDescriptor when table is modified.",
|
||||
regionServerGroup.isPresent());
|
||||
assertEquals("rsGroup2", regionServerGroup.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTableShouldNotAddedIntoRSGroup_WhenModifyTableFails() throws Exception {
|
||||
RSGroupInfo rsGroup1 = addGroup("rsGroup1", 1);
|
||||
assertEquals(0, rsGroup1.getTables().size());
|
||||
|
||||
createTableWithRSGroupDetail(rsGroup1.getName());
|
||||
TableDescriptor descriptor = admin.getConnection().getTable(tableName).getDescriptor();
|
||||
|
||||
RSGroupInfo rsGroup2 = addGroup("rsGroup2", 1);
|
||||
final TableDescriptor newTableDescriptor =
|
||||
TableDescriptorBuilder.newBuilder(descriptor).setRegionServerGroup(rsGroup2.getName())
|
||||
.removeColumnFamily(familyNameBytes).build();
|
||||
|
||||
// Removed family to fail pre-check validation
|
||||
try {
|
||||
admin.modifyTable(newTableDescriptor);
|
||||
fail("Should have thrown DoNotRetryIOException but no exception thrown.");
|
||||
} catch (DoNotRetryIOException e) {
|
||||
assertTrue(
|
||||
e.getCause().getMessage().contains("Table should have at least one column family"));
|
||||
}
|
||||
rsGroup2 = rsGroupAdmin.getRSGroupInfo(rsGroup2.getName());
|
||||
assertEquals("Table must not have moved to RSGroup as table modify failed", 0,
|
||||
rsGroup2.getTables().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTableShouldNotAddedIntoRSGroup_WhenTableCreationFails() throws Exception {
|
||||
RSGroupInfo rsGroup1 = addGroup("rsGroup1", 1);
|
||||
assertEquals(0, rsGroup1.getTables().size());
|
||||
|
||||
// Create TableDescriptor without a family so creation fails
|
||||
TableDescriptor desc =
|
||||
TableDescriptorBuilder.newBuilder(tableName).setRegionServerGroup(rsGroup1.getName())
|
||||
.build();
|
||||
try {
|
||||
admin.createTable(desc, getSpitKeys(5));
|
||||
fail("Should have thrown DoNotRetryIOException but no exception thrown.");
|
||||
} catch (DoNotRetryIOException e) {
|
||||
assertTrue(
|
||||
e.getCause().getMessage().contains("Table should have at least one column family"));
|
||||
}
|
||||
rsGroup1 = rsGroupAdmin.getRSGroupInfo(rsGroup1.getName());
|
||||
assertEquals("Table must not have moved to RSGroup as table create operation failed", 0,
|
||||
rsGroup1.getTables().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSystemTablesCanBeMovedToNewRSGroupByModifyingTable() throws Exception {
|
||||
RSGroupInfo rsGroup1 = addGroup("rsGroup1", 1);
|
||||
assertEquals(0, rsGroup1.getTables().size());
|
||||
final TableName tableName = TableName.valueOf("hbase:meta");
|
||||
final TableDescriptor descriptor = admin.getConnection().getTable(tableName).getDescriptor();
|
||||
final TableDescriptor newTableDescriptor =
|
||||
TableDescriptorBuilder.newBuilder(descriptor).setRegionServerGroup(rsGroup1.getName())
|
||||
.build();
|
||||
admin.modifyTable(newTableDescriptor);
|
||||
final RSGroupInfo rsGroupInfoOfTable = rsGroupAdmin.getRSGroupInfoOfTable(tableName);
|
||||
assertEquals(rsGroup1.getName(), rsGroupInfoOfTable.getName());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUpdateTableDescriptorOnlyIfRSGroupInfoWasStoredInTableDescriptor()
|
||||
throws Exception {
|
||||
RSGroupInfo rsGroup1 = addGroup("rsGroup1", 1);
|
||||
assertEquals(0, rsGroup1.getTables().size());
|
||||
// create table with rs group info stored in table descriptor
|
||||
createTable(tableName, rsGroup1.getName());
|
||||
final TableName table2 = TableName.valueOf(tableName.getNameAsString() + "_2");
|
||||
// create table with no rs group info
|
||||
createTable(table2, null);
|
||||
rsGroupAdmin.moveTables(Sets.newHashSet(tableName), rsGroup1.getName());
|
||||
assertTrue("RSGroup info is not updated into TableDescriptor when table created",
|
||||
admin.getConnection().getTable(tableName).getDescriptor().getRegionServerGroup()
|
||||
.isPresent());
|
||||
assertFalse("Table descriptor should not have been updated "
|
||||
+ "as rs group info was not stored in table descriptor.",
|
||||
admin.getConnection().getTable(table2).getDescriptor().getRegionServerGroup().isPresent());
|
||||
|
||||
final String rsGroup2 = "rsGroup2";
|
||||
rsGroupAdmin.renameRSGroup(rsGroup1.getName(), rsGroup2);
|
||||
assertEquals(rsGroup2,
|
||||
admin.getConnection().getTable(tableName).getDescriptor().getRegionServerGroup().get());
|
||||
assertFalse("Table descriptor should not have been updated "
|
||||
+ "as rs group info was not stored in table descriptor.",
|
||||
admin.getConnection().getTable(table2).getDescriptor().getRegionServerGroup().isPresent());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testModifyAndMoveTableScenario() throws Exception {
|
||||
RSGroupInfo rsGroup1 = addGroup("rsGroup1", 1);
|
||||
assertEquals(0, rsGroup1.getTables().size());
|
||||
// create table with rs group info stored in table descriptor
|
||||
createTable(tableName, rsGroup1.getName());
|
||||
final TableName table2 = TableName.valueOf(tableName.getNameAsString() + "_2");
|
||||
// create table with no rs group info
|
||||
createTable(table2, null);
|
||||
rsGroupAdmin.moveTables(Sets.newHashSet(table2), rsGroup1.getName());
|
||||
|
||||
RSGroupInfo rsGroup2 = addGroup("rsGroup2", 1);
|
||||
rsGroupAdmin.moveTables(Sets.newHashSet(tableName, table2), rsGroup2.getName());
|
||||
rsGroup2 = rsGroupAdmin.getRSGroupInfo(rsGroup2.getName());
|
||||
assertEquals("Table movement failed.", 2, rsGroup2.getTables().size());
|
||||
}
|
||||
|
||||
private void createTable(TableName tName, String rsGroupName) throws Exception {
|
||||
ColumnFamilyDescriptor f1 = ColumnFamilyDescriptorBuilder.newBuilder(familyNameBytes).build();
|
||||
final TableDescriptorBuilder builder =
|
||||
TableDescriptorBuilder.newBuilder(tName).setColumnFamily(f1);
|
||||
if (rsGroupName != null) {
|
||||
builder.setRegionServerGroup(rsGroupName);
|
||||
}
|
||||
TableDescriptor desc = builder.build();
|
||||
admin.createTable(desc, getSpitKeys(10));
|
||||
TEST_UTIL.waitFor(WAIT_TIMEOUT, (Waiter.Predicate<Exception>) () -> {
|
||||
List<String> regions = getTableRegionMap().get(tName);
|
||||
if (regions == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return getTableRegionMap().get(tName).size() >= 5;
|
||||
});
|
||||
}
|
||||
|
||||
private byte[][] getSpitKeys(int numRegions) throws IOException {
|
||||
if (numRegions < 3) {
|
||||
throw new IOException("Must create at least 3 regions");
|
||||
}
|
||||
byte[] startKey = Bytes.toBytes("aaaaa");
|
||||
byte[] endKey = Bytes.toBytes("zzzzz");
|
||||
return Bytes.split(startKey, endKey, numRegions - 3);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue