HBASE-21707 Fix warnings in hbase-rsgroup module and also make the UTs more stable
Signed-off-by: Zheng Hu <openinx@gmail.com>
This commit is contained in:
zhangduo
Duo Zhang
parent
fb91f64a04
commit
5c88976fa3
|
|
@ -27,7 +27,6 @@ import java.util.LinkedList;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hadoop.hbase.NamespaceDescriptor;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
|
|
@ -451,10 +450,10 @@ public class RSGroupAdminServer implements RSGroupAdmin {
|
|||
"the RSGroup can be removed.");
|
||||
}
|
||||
for (NamespaceDescriptor ns : master.getClusterSchema().getNamespaces()) {
|
||||
String nsGroup = ns.getConfigurationValue(rsGroupInfo.NAMESPACE_DESC_PROP_GROUP);
|
||||
String nsGroup = ns.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP);
|
||||
if (nsGroup != null && nsGroup.equals(name)) {
|
||||
throw new ConstraintException("RSGroup " + name + " is referenced by namespace: " +
|
||||
ns.getName());
|
||||
throw new ConstraintException(
|
||||
"RSGroup " + name + " is referenced by namespace: " + ns.getName());
|
||||
}
|
||||
}
|
||||
rsGroupInfoManager.removeRSGroup(name);
|
||||
|
|
|
|||
|
|
@ -172,6 +172,7 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
|
|||
ListMultimap<String,RegionInfo> regionMap = ArrayListMultimap.create();
|
||||
ListMultimap<String,ServerName> serverMap = ArrayListMultimap.create();
|
||||
generateGroupMaps(regions, servers, regionMap, serverMap);
|
||||
LOG.info("=======" + regionMap + ", " + serverMap);
|
||||
for(String groupKey : regionMap.keySet()) {
|
||||
if (regionMap.get(groupKey).size() > 0) {
|
||||
Map<ServerName, List<RegionInfo>> result =
|
||||
|
|
@ -189,6 +190,7 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
|
|||
}
|
||||
}
|
||||
}
|
||||
LOG.info("=======" + assignments);
|
||||
return assignments;
|
||||
}
|
||||
|
||||
|
|
@ -296,7 +298,7 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer {
|
|||
return filterServers(RSGroupInfo.getServers(), onlineServers);
|
||||
} else {
|
||||
LOG.warn("RSGroup Information found to be null. Some regions might be unassigned.");
|
||||
return Collections.EMPTY_LIST;
|
||||
return Collections.emptyList();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -31,14 +31,14 @@ import java.util.Map;
|
|||
import java.util.Set;
|
||||
import java.util.TreeMap;
|
||||
import java.util.TreeSet;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.TableDescriptors;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.client.RegionInfo;
|
||||
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptor;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.master.HMaster;
|
||||
import org.apache.hadoop.hbase.master.MasterServices;
|
||||
import org.apache.hadoop.hbase.master.RegionPlan;
|
||||
|
|
@ -47,7 +47,6 @@ import org.apache.hadoop.hbase.net.Address;
|
|||
import org.apache.hadoop.hbase.rsgroup.RSGroupInfo;
|
||||
import org.apache.hadoop.hbase.rsgroup.RSGroupInfoManager;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.invocation.InvocationOnMock;
|
||||
import org.mockito.stubbing.Answer;
|
||||
|
|
@ -71,7 +70,7 @@ public class RSGroupableBalancerTestBase {
|
|||
static List<ServerName> servers;
|
||||
static Map<String, RSGroupInfo> groupMap;
|
||||
static Map<TableName, String> tableMap = new HashMap<>();
|
||||
static List<HTableDescriptor> tableDescs;
|
||||
static List<TableDescriptor> tableDescs;
|
||||
int[] regionAssignment = new int[] { 2, 5, 7, 10, 4, 3, 1 };
|
||||
static int regionId = 0;
|
||||
|
||||
|
|
@ -390,11 +389,11 @@ public class RSGroupableBalancerTestBase {
|
|||
* @param hasBogusTable there is a table that does not determine the group
|
||||
* @return the list of table descriptors
|
||||
*/
|
||||
protected static List<HTableDescriptor> constructTableDesc(boolean hasBogusTable) {
|
||||
List<HTableDescriptor> tds = Lists.newArrayList();
|
||||
protected static List<TableDescriptor> constructTableDesc(boolean hasBogusTable) {
|
||||
List<TableDescriptor> tds = Lists.newArrayList();
|
||||
int index = rand.nextInt(groups.length);
|
||||
for (int i = 0; i < tables.length; i++) {
|
||||
HTableDescriptor htd = new HTableDescriptor(tables[i]);
|
||||
TableDescriptor htd = TableDescriptorBuilder.newBuilder(tables[i]).build();
|
||||
int grpIndex = (i + index) % groups.length;
|
||||
String groupName = groups[grpIndex];
|
||||
tableMap.put(tables[i], groupName);
|
||||
|
|
@ -402,7 +401,7 @@ public class RSGroupableBalancerTestBase {
|
|||
}
|
||||
if (hasBogusTable) {
|
||||
tableMap.put(table0, "");
|
||||
tds.add(new HTableDescriptor(table0));
|
||||
tds.add(TableDescriptorBuilder.newBuilder(table0).build());
|
||||
}
|
||||
return tds;
|
||||
}
|
||||
|
|
@ -452,7 +451,7 @@ public class RSGroupableBalancerTestBase {
|
|||
}
|
||||
}
|
||||
|
||||
for(HTableDescriptor desc : tableDescs){
|
||||
for(TableDescriptor desc : tableDescs){
|
||||
if(gm.getRSGroupOfTable(desc.getTableName()).endsWith(groupOfServer.getName())){
|
||||
tableName = desc.getTableName();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -109,8 +109,7 @@ public class TestRSGroupBasedLoadBalancer extends RSGroupableBalancerTestBase {
|
|||
.roundRobinAssignment(regions, servers);
|
||||
//test empty region/servers scenario
|
||||
//this should not throw an NPE
|
||||
loadBalancer.roundRobinAssignment(regions,
|
||||
Collections.EMPTY_LIST);
|
||||
loadBalancer.roundRobinAssignment(regions, Collections.emptyList());
|
||||
//test regular scenario
|
||||
assertTrue(assignments.keySet().size() == servers.size());
|
||||
for (ServerName sn : assignments.keySet()) {
|
||||
|
|
|
|||
|
|
@ -28,23 +28,24 @@ import java.util.Iterator;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HColumnDescriptor;
|
||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||
import org.apache.hadoop.hbase.NamespaceDescriptor;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.TableNotFoundException;
|
||||
import org.apache.hadoop.hbase.Waiter;
|
||||
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
|
||||
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptor;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.constraint.ConstraintException;
|
||||
import org.apache.hadoop.hbase.master.ServerManager;
|
||||
import org.apache.hadoop.hbase.master.TableNamespaceManager;
|
||||
import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
|
||||
import org.apache.hadoop.hbase.net.Address;
|
||||
import org.apache.hadoop.hbase.quotas.QuotaUtil;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Assert;
|
||||
|
|
@ -56,7 +57,6 @@ import org.junit.experimental.categories.Category;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
|
||||
|
||||
@Category({ MediumTests.class })
|
||||
|
|
@ -135,9 +135,9 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase {
|
|||
}
|
||||
|
||||
try {
|
||||
admin.setBalancerRunning(true,true);
|
||||
admin.balancerSwitch(true, true);
|
||||
rsGroupAdmin.balanceRSGroup("bogus");
|
||||
admin.setBalancerRunning(false,true);
|
||||
admin.balancerSwitch(false, true);
|
||||
fail("Expected move with bogus group to fail");
|
||||
} catch (ConstraintException ex) {
|
||||
// expected
|
||||
|
|
@ -154,8 +154,7 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase {
|
|||
assertTrue(observer.postAddRSGroupCalled);
|
||||
|
||||
admin.createNamespace(NamespaceDescriptor.create(nsName)
|
||||
.addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, groupName)
|
||||
.build());
|
||||
.addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, groupName).build());
|
||||
// test removing a referenced group
|
||||
try {
|
||||
rsGroupAdmin.removeRSGroup(groupName);
|
||||
|
|
@ -164,10 +163,8 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase {
|
|||
}
|
||||
// test modify group
|
||||
// changing with the same name is fine
|
||||
admin.modifyNamespace(
|
||||
NamespaceDescriptor.create(nsName)
|
||||
.addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, groupName)
|
||||
.build());
|
||||
admin.modifyNamespace(NamespaceDescriptor.create(nsName)
|
||||
.addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, groupName).build());
|
||||
String anotherGroup = tablePrefix + "_anotherGroup";
|
||||
rsGroupAdmin.addRSGroup(anotherGroup);
|
||||
// test add non-existent group
|
||||
|
|
@ -177,8 +174,7 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase {
|
|||
assertTrue(observer.postRemoveRSGroupCalled);
|
||||
try {
|
||||
admin.createNamespace(NamespaceDescriptor.create(nsName)
|
||||
.addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "foo")
|
||||
.build());
|
||||
.addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "foo").build());
|
||||
fail("Expected a constraint exception");
|
||||
} catch (IOException ex) {
|
||||
}
|
||||
|
|
@ -249,8 +245,8 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase {
|
|||
if (regionsB == null) {
|
||||
return false;
|
||||
}
|
||||
return getTableRegionMap().get(tableNameA).size() >= 1
|
||||
&& getTableRegionMap().get(tableNameB).size() >= 1;
|
||||
return getTableRegionMap().get(tableNameA).size() >= 1 &&
|
||||
getTableRegionMap().get(tableNameB).size() >= 1;
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -332,8 +328,8 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase {
|
|||
admin.disableTable(tableName);
|
||||
admin.truncateTable(tableName, true);
|
||||
Assert.assertEquals(1, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size());
|
||||
Assert.assertEquals(tableName, rsGroupAdmin.getRSGroupInfo(
|
||||
newGroup.getName()).getTables().first());
|
||||
Assert.assertEquals(tableName,
|
||||
rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().first());
|
||||
|
||||
// verify removed table is removed from group
|
||||
TEST_UTIL.deleteTable(tableName);
|
||||
|
|
@ -396,8 +392,7 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase {
|
|||
}
|
||||
|
||||
try {
|
||||
rsGroupAdmin.moveServersAndTables(
|
||||
Sets.newHashSet(Address.fromParts("bogus",123)),
|
||||
rsGroupAdmin.moveServersAndTables(Sets.newHashSet(Address.fromParts("bogus", 123)),
|
||||
Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
|
||||
fail("Table " + tableName + " shouldn't have been successfully moved.");
|
||||
} catch (IOException ex) {
|
||||
|
|
@ -412,25 +407,23 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase {
|
|||
toggleQuotaCheckAndRestartMiniCluster(true);
|
||||
String nsp = "np1";
|
||||
NamespaceDescriptor nspDesc =
|
||||
NamespaceDescriptor.create(nsp)
|
||||
.addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
|
||||
NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5")
|
||||
.addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build();
|
||||
admin.createNamespace(nspDesc);
|
||||
assertEquals(3, admin.listNamespaceDescriptors().length);
|
||||
HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
|
||||
HTableDescriptor tableDescOne =
|
||||
new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"));
|
||||
tableDescOne.addFamily(fam1);
|
||||
ColumnFamilyDescriptor fam1 = ColumnFamilyDescriptorBuilder.of("fam1");
|
||||
TableDescriptor tableDescOne = TableDescriptorBuilder
|
||||
.newBuilder(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"))
|
||||
.setColumnFamily(fam1).build();
|
||||
admin.createTable(tableDescOne);
|
||||
|
||||
HTableDescriptor tableDescTwo =
|
||||
new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"));
|
||||
tableDescTwo.addFamily(fam1);
|
||||
TableDescriptor tableDescTwo = TableDescriptorBuilder
|
||||
.newBuilder(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"))
|
||||
.setColumnFamily(fam1).build();
|
||||
boolean constraintViolated = false;
|
||||
|
||||
try {
|
||||
admin.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"),
|
||||
6);
|
||||
admin.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 6);
|
||||
Assert.fail("Creation table should fail because of quota violation.");
|
||||
} catch (Exception exp) {
|
||||
assertTrue(exp instanceof IOException);
|
||||
|
|
@ -463,8 +456,7 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase {
|
|||
TEST_UTIL.shutdownMiniCluster();
|
||||
TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, enable);
|
||||
TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1);
|
||||
TEST_UTIL.getConfiguration().setInt(
|
||||
ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
|
||||
TEST_UTIL.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
|
||||
NUM_SLAVES_BASE - 1);
|
||||
TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
|
||||
initialize();
|
||||
|
|
|
|||
|
|
@ -29,18 +29,16 @@ import java.util.Iterator;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.hadoop.hbase.ClusterMetrics.Option;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.Waiter;
|
||||
import org.apache.hadoop.hbase.client.ClusterConnection;
|
||||
import org.apache.hadoop.hbase.client.RegionInfo;
|
||||
import org.apache.hadoop.hbase.constraint.ConstraintException;
|
||||
import org.apache.hadoop.hbase.net.Address;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Assert;
|
||||
|
|
@ -52,10 +50,6 @@ import org.junit.experimental.categories.Category;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
|
||||
|
||||
@Category({ MediumTests.class })
|
||||
|
|
@ -106,8 +100,7 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
});
|
||||
|
||||
// get target region to move
|
||||
Map<ServerName,List<String>> assignMap =
|
||||
getTableServerRegionMap().get(tableName);
|
||||
Map<ServerName, List<String>> assignMap = getTableServerRegionMap().get(tableName);
|
||||
String targetRegion = null;
|
||||
for (ServerName server : assignMap.keySet()) {
|
||||
targetRegion = assignMap.get(server).size() > 0 ? assignMap.get(server).get(0) : null;
|
||||
|
|
@ -116,36 +109,32 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
}
|
||||
}
|
||||
// get server which is not a member of new group
|
||||
ServerName targetServer = null;
|
||||
ServerName tmpTargetServer = null;
|
||||
for (ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
|
||||
.getLiveServerMetrics().keySet()) {
|
||||
if (!newGroup.containsServer(server.getAddress())) {
|
||||
targetServer = server;
|
||||
tmpTargetServer = server;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
final AdminProtos.AdminService.BlockingInterface targetRS =
|
||||
((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
|
||||
|
||||
final ServerName targetServer = tmpTargetServer;
|
||||
// move target server to group
|
||||
rsGroupAdmin.moveServers(Sets.newHashSet(targetServer.getAddress()),
|
||||
newGroup.getName());
|
||||
rsGroupAdmin.moveServers(Sets.newHashSet(targetServer.getAddress()), newGroup.getName());
|
||||
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
|
||||
@Override
|
||||
public boolean evaluate() throws Exception {
|
||||
return ProtobufUtil.getOnlineRegions(targetRS).size() <= 0;
|
||||
return admin.getRegions(targetServer).size() <= 0;
|
||||
}
|
||||
});
|
||||
|
||||
// Lets move this region to the new group.
|
||||
TEST_UTIL.getAdmin().move(Bytes.toBytes(RegionInfo.encodeRegionName(
|
||||
Bytes.toBytes(targetRegion))), Bytes.toBytes(targetServer.getServerName()));
|
||||
TEST_UTIL.getAdmin().move(
|
||||
Bytes.toBytes(RegionInfo.encodeRegionName(Bytes.toBytes(targetRegion))),
|
||||
Bytes.toBytes(targetServer.getServerName()));
|
||||
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
|
||||
@Override
|
||||
public boolean evaluate() throws Exception {
|
||||
return
|
||||
getTableRegionMap().get(tableName) != null &&
|
||||
return getTableRegionMap().get(tableName) != null &&
|
||||
getTableRegionMap().get(tableName).size() == 6 &&
|
||||
admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION))
|
||||
.getRegionStatesInTransition().size() < 1;
|
||||
|
|
@ -153,7 +142,7 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
});
|
||||
|
||||
// verify that targetServer didn't open it
|
||||
for (RegionInfo region: ProtobufUtil.getOnlineRegions(targetRS)) {
|
||||
for (RegionInfo region : admin.getRegions(targetServer)) {
|
||||
if (targetRegion.equals(region.getRegionNameAsString())) {
|
||||
fail("Target server opened region");
|
||||
}
|
||||
|
|
@ -161,8 +150,7 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testRegionServerMove() throws IOException,
|
||||
InterruptedException {
|
||||
public void testRegionServerMove() throws IOException, InterruptedException {
|
||||
int initNumGroups = rsGroupAdmin.listRSGroups().size();
|
||||
RSGroupInfo appInfo = addGroup(getGroupName(name.getMethodName()), 1);
|
||||
RSGroupInfo adminInfo = addGroup(getGroupName(name.getMethodName()), 1);
|
||||
|
|
@ -171,11 +159,9 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
assertEquals(1, adminInfo.getServers().size());
|
||||
assertEquals(1, appInfo.getServers().size());
|
||||
assertEquals(getNumServers() - 2, dInfo.getServers().size());
|
||||
rsGroupAdmin.moveServers(appInfo.getServers(),
|
||||
RSGroupInfo.DEFAULT_GROUP);
|
||||
rsGroupAdmin.moveServers(appInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
|
||||
rsGroupAdmin.removeRSGroup(appInfo.getName());
|
||||
rsGroupAdmin.moveServers(adminInfo.getServers(),
|
||||
RSGroupInfo.DEFAULT_GROUP);
|
||||
rsGroupAdmin.moveServers(adminInfo.getServers(), RSGroupInfo.DEFAULT_GROUP);
|
||||
rsGroupAdmin.removeRSGroup(adminInfo.getName());
|
||||
Assert.assertEquals(rsGroupAdmin.listRSGroups().size(), initNumGroups);
|
||||
}
|
||||
|
|
@ -216,8 +202,8 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
|
||||
@Override
|
||||
public boolean evaluate() throws Exception {
|
||||
return getNumServers() ==
|
||||
rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size();
|
||||
return getNumServers() == rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
|
||||
.getServers().size();
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -238,15 +224,15 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
LOG.info("testRemoveServers");
|
||||
final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 3);
|
||||
Iterator<Address> iterator = newGroup.getServers().iterator();
|
||||
ServerName targetServer = ServerName.parseServerName(iterator.next().toString());
|
||||
ServerName targetServer = getServerName(iterator.next());
|
||||
|
||||
// remove online servers
|
||||
try {
|
||||
rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
|
||||
fail("Online servers shouldn't have been successfully removed.");
|
||||
} catch (IOException ex) {
|
||||
String exp = "Server " + targetServer.getAddress()
|
||||
+ " is an online server, not allowed to remove.";
|
||||
String exp =
|
||||
"Server " + targetServer.getAddress() + " is an online server, not allowed to remove.";
|
||||
String msg = "Expected '" + exp + "' in exception message: ";
|
||||
assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
|
||||
}
|
||||
|
|
@ -254,16 +240,11 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
|
||||
// remove dead servers
|
||||
NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size();
|
||||
AdminProtos.AdminService.BlockingInterface targetRS =
|
||||
((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
|
||||
try {
|
||||
targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null,
|
||||
GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName());
|
||||
// stopping may cause an exception
|
||||
// due to the connection loss
|
||||
LOG.info("stopping server " + targetServer.getServerName());
|
||||
targetRS.stopServer(null,
|
||||
AdminProtos.StopServerRequest.newBuilder().setReason("Die").build());
|
||||
admin.stopRegionServer(targetServer.getAddress().toString());
|
||||
NUM_DEAD_SERVERS++;
|
||||
} catch (Exception e) {
|
||||
}
|
||||
|
|
@ -272,8 +253,8 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
|
||||
@Override
|
||||
public boolean evaluate() throws Exception {
|
||||
return !master.getServerManager().areDeadServersInProgress()
|
||||
&& cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS;
|
||||
return !master.getServerManager().areDeadServersInProgress() &&
|
||||
cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS;
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -281,8 +262,8 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress()));
|
||||
fail("Dead servers shouldn't have been successfully removed.");
|
||||
} catch (IOException ex) {
|
||||
String exp = "Server " + targetServer.getAddress() + " is on the dead servers list,"
|
||||
+ " Maybe it will come back again, not allowed to remove.";
|
||||
String exp = "Server " + targetServer.getAddress() + " is on the dead servers list," +
|
||||
" Maybe it will come back again, not allowed to remove.";
|
||||
String msg = "Expected '" + exp + "' in exception message: ";
|
||||
assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
|
||||
}
|
||||
|
|
@ -290,10 +271,7 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
|
||||
// remove decommissioned servers
|
||||
List<ServerName> serversToDecommission = new ArrayList<>();
|
||||
targetServer = ServerName.parseServerName(iterator.next().toString());
|
||||
targetRS = ((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
|
||||
targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null,
|
||||
GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName());
|
||||
targetServer = getServerName(iterator.next());
|
||||
assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer));
|
||||
serversToDecommission.add(targetServer);
|
||||
|
||||
|
|
@ -363,8 +341,8 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
|
||||
fail("servers shouldn't have been successfully moved.");
|
||||
} catch (IOException ex) {
|
||||
String exp = "Target RSGroup " + RSGroupInfo.DEFAULT_GROUP +
|
||||
" is same as source " + RSGroupInfo.DEFAULT_GROUP + " RSGroup.";
|
||||
String exp = "Target RSGroup " + RSGroupInfo.DEFAULT_GROUP + " is same as source " +
|
||||
RSGroupInfo.DEFAULT_GROUP + " RSGroup.";
|
||||
String msg = "Expected '" + exp + "' in exception message: ";
|
||||
assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
|
||||
}
|
||||
|
|
@ -376,10 +354,8 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size());
|
||||
|
||||
// verify new group info
|
||||
Assert.assertEquals(1,
|
||||
rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers().size());
|
||||
Assert.assertEquals(0,
|
||||
rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size());
|
||||
Assert.assertEquals(1, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers().size());
|
||||
Assert.assertEquals(0, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size());
|
||||
|
||||
// get all region to move targetServer
|
||||
List<String> regionList = getTableRegionMap().get(tableName);
|
||||
|
|
@ -413,8 +389,8 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName());
|
||||
|
||||
// verify servers' not exist in old group
|
||||
Set<Address> defaultServers = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
|
||||
.getServers();
|
||||
Set<Address> defaultServers =
|
||||
rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers();
|
||||
assertFalse(defaultServers.contains(targetServer.getAddress()));
|
||||
|
||||
// verify servers' exist in new group
|
||||
|
|
@ -422,8 +398,8 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
assertTrue(newGroupServers.contains(targetServer.getAddress()));
|
||||
|
||||
// verify tables' not exist in old group
|
||||
Set<TableName> defaultTables = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
|
||||
.getTables();
|
||||
Set<TableName> defaultTables =
|
||||
rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables();
|
||||
assertFalse(defaultTables.contains(tableName));
|
||||
|
||||
// verify tables' exist in new group
|
||||
|
|
@ -451,8 +427,8 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
rsGroupAdmin.moveServers(defaultGroup.getServers(), fooGroup.getName());
|
||||
fail(RSGroupAdminServer.KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE);
|
||||
} catch (ConstraintException ex) {
|
||||
assertTrue(ex.getMessage().contains(RSGroupAdminServer
|
||||
.KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE));
|
||||
assertTrue(
|
||||
ex.getMessage().contains(RSGroupAdminServer.KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE));
|
||||
}
|
||||
|
||||
// test success case, remove one server from default ,keep at least one server
|
||||
|
|
@ -470,8 +446,8 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase {
|
|||
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
|
||||
@Override
|
||||
public boolean evaluate() throws Exception {
|
||||
return getNumServers() ==
|
||||
rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size();
|
||||
return getNumServers() == rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP)
|
||||
.getServers().size();
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -23,18 +23,18 @@ import static org.junit.Assert.assertTrue;
|
|||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HColumnDescriptor;
|
||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||
import org.apache.hadoop.hbase.NamespaceDescriptor;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.Waiter;
|
||||
import org.apache.hadoop.hbase.Waiter.Predicate;
|
||||
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.client.RegionInfo;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptor;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
|
|
@ -45,7 +45,6 @@ import org.junit.experimental.categories.Category;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
|
||||
|
||||
@Category({ MediumTests.class })
|
||||
|
|
@ -81,15 +80,13 @@ public class TestRSGroupsBalance extends TestRSGroupsBase {
|
|||
public void testGroupBalance() throws Exception {
|
||||
LOG.info(name.getMethodName());
|
||||
String newGroupName = getGroupName(name.getMethodName());
|
||||
final RSGroupInfo newGroup = addGroup(newGroupName, 3);
|
||||
addGroup(newGroupName, 3);
|
||||
|
||||
final TableName tableName = TableName.valueOf(tablePrefix + "_ns", name.getMethodName());
|
||||
admin.createNamespace(
|
||||
NamespaceDescriptor.create(tableName.getNamespaceAsString())
|
||||
admin.createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString())
|
||||
.addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, newGroupName).build());
|
||||
final byte[] familyNameBytes = Bytes.toBytes("f");
|
||||
final HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||
desc.addFamily(new HColumnDescriptor("f"));
|
||||
final TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
|
||||
.setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
|
||||
byte[] startKey = Bytes.toBytes("aaaaa");
|
||||
byte[] endKey = Bytes.toBytes("zzzzz");
|
||||
admin.createTable(desc, startKey, endKey, 6);
|
||||
|
|
@ -105,10 +102,9 @@ public class TestRSGroupsBalance extends TestRSGroupsBase {
|
|||
});
|
||||
|
||||
// make assignment uneven, move all regions to one server
|
||||
Map<ServerName,List<String>> assignMap =
|
||||
getTableServerRegionMap().get(tableName);
|
||||
Map<ServerName, List<String>> assignMap = getTableServerRegionMap().get(tableName);
|
||||
final ServerName first = assignMap.entrySet().iterator().next().getKey();
|
||||
for(RegionInfo region: admin.getTableRegions(tableName)) {
|
||||
for (RegionInfo region : admin.getRegions(tableName)) {
|
||||
if (!assignMap.get(first).contains(region.getRegionNameAsString())) {
|
||||
admin.move(region.getEncodedNameAsBytes(), Bytes.toBytes(first.getServerName()));
|
||||
}
|
||||
|
|
@ -129,17 +125,17 @@ public class TestRSGroupsBalance extends TestRSGroupsBase {
|
|||
});
|
||||
|
||||
// balance the other group and make sure it doesn't affect the new group
|
||||
admin.setBalancerRunning(true,true);
|
||||
admin.balancerSwitch(true, true);
|
||||
rsGroupAdmin.balanceRSGroup(RSGroupInfo.DEFAULT_GROUP);
|
||||
assertEquals(6, getTableServerRegionMap().get(tableName).get(first).size());
|
||||
|
||||
// disable balance, balancer will not be run and return false
|
||||
admin.setBalancerRunning(false,true);
|
||||
admin.balancerSwitch(false, true);
|
||||
assertFalse(rsGroupAdmin.balanceRSGroup(newGroupName));
|
||||
assertEquals(6, getTableServerRegionMap().get(tableName).get(first).size());
|
||||
|
||||
// enable balance
|
||||
admin.setBalancerRunning(true,true);
|
||||
admin.balancerSwitch(true, true);
|
||||
rsGroupAdmin.balanceRSGroup(newGroupName);
|
||||
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
|
||||
@Override
|
||||
|
|
@ -152,7 +148,7 @@ public class TestRSGroupsBalance extends TestRSGroupsBase {
|
|||
return true;
|
||||
}
|
||||
});
|
||||
admin.setBalancerRunning(false,true);
|
||||
admin.balancerSwitch(false, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -165,12 +161,12 @@ public class TestRSGroupsBalance extends TestRSGroupsBase {
|
|||
TEST_UTIL.createMultiRegionTable(tableName, new byte[] { 'f' }, 15);
|
||||
TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
|
||||
|
||||
rsGroupAdminEndpoint.getGroupInfoManager()
|
||||
.moveTables(Sets.newHashSet(tableName), RSGroupInfo.getName());
|
||||
rsGroupAdminEndpoint.getGroupInfoManager().moveTables(Sets.newHashSet(tableName),
|
||||
RSGroupInfo.getName());
|
||||
|
||||
admin.setBalancerRunning(true,true);
|
||||
admin.balancerSwitch(true, true);
|
||||
assertTrue(rsGroupAdmin.balanceRSGroup(RSGroupInfo.getName()));
|
||||
admin.setBalancerRunning(false,true);
|
||||
admin.balancerSwitch(false, true);
|
||||
assertTrue(observer.preBalanceRSGroupCalled);
|
||||
assertTrue(observer.postBalanceRSGroupCalled);
|
||||
|
||||
|
|
@ -179,8 +175,7 @@ public class TestRSGroupsBalance extends TestRSGroupsBase {
|
|||
public boolean evaluate() throws Exception {
|
||||
ServerName serverName =
|
||||
ServerName.valueOf(RSGroupInfo.getServers().iterator().next().toString(), 1);
|
||||
return admin.getConnection().getAdmin()
|
||||
.getOnlineRegions(serverName).size() == 15;
|
||||
return admin.getConnection().getAdmin().getRegions(serverName).size() == 15;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import java.util.Optional;
|
|||
import java.util.Random;
|
||||
import java.util.Set;
|
||||
import java.util.TreeMap;
|
||||
import java.util.regex.Pattern;
|
||||
import org.apache.hadoop.hbase.ClusterMetrics;
|
||||
import org.apache.hadoop.hbase.ClusterMetrics.Option;
|
||||
import org.apache.hadoop.hbase.HBaseCluster;
|
||||
|
|
@ -81,7 +82,7 @@ public abstract class TestRSGroupsBase {
|
|||
protected static RSGroupAdminEndpoint rsGroupAdminEndpoint;
|
||||
protected static CPMasterObserver observer;
|
||||
|
||||
public final static long WAIT_TIMEOUT = 60000*5;
|
||||
public final static long WAIT_TIMEOUT = 60000;
|
||||
public final static int NUM_SLAVES_BASE = 4; //number of slaves for the smallest cluster
|
||||
public static int NUM_DEAD_SERVERS = 0;
|
||||
|
||||
|
|
@ -110,7 +111,7 @@ public abstract class TestRSGroupsBase {
|
|||
protected static void initialize() throws Exception {
|
||||
admin = TEST_UTIL.getAdmin();
|
||||
cluster = TEST_UTIL.getHBaseCluster();
|
||||
master = ((MiniHBaseCluster)cluster).getMaster();
|
||||
master = TEST_UTIL.getMiniHBaseCluster().getMaster();
|
||||
|
||||
//wait for balancer to come online
|
||||
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
|
||||
|
|
@ -120,7 +121,7 @@ public abstract class TestRSGroupsBase {
|
|||
((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline();
|
||||
}
|
||||
});
|
||||
admin.setBalancerRunning(false,true);
|
||||
admin.balancerSwitch(false, true);
|
||||
rsGroupAdmin = new VerifyingRSGroupAdminClient(
|
||||
new RSGroupAdminClient(TEST_UTIL.getConnection()), TEST_UTIL.getConfiguration());
|
||||
MasterCoprocessorHost host = master.getMasterCoprocessorHost();
|
||||
|
|
@ -206,7 +207,8 @@ public abstract class TestRSGroupsBase {
|
|||
}
|
||||
|
||||
public void deleteTableIfNecessary() throws IOException {
|
||||
for (TableDescriptor desc : TEST_UTIL.getAdmin().listTables(tablePrefix+".*")) {
|
||||
for (TableDescriptor desc : TEST_UTIL.getAdmin()
|
||||
.listTableDescriptors(Pattern.compile(tablePrefix + ".*"))) {
|
||||
TEST_UTIL.deleteTable(desc.getTableName());
|
||||
}
|
||||
}
|
||||
|
|
@ -290,6 +292,16 @@ public abstract class TestRSGroupsBase {
|
|||
return groupPrefix + "_" + baseName + "_" + rand.nextInt(Integer.MAX_VALUE);
|
||||
}
|
||||
|
||||
/**
|
||||
* The server name in group does not contain the start code, this method will find out the start
|
||||
* code and construct the ServerName object.
|
||||
*/
|
||||
protected ServerName getServerName(Address addr) {
|
||||
return TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream()
|
||||
.map(t -> t.getRegionServer().getServerName()).filter(sn -> sn.getAddress().equals(addr))
|
||||
.findFirst().get();
|
||||
}
|
||||
|
||||
public static class CPMasterObserver implements MasterCoprocessor, MasterObserver {
|
||||
boolean preBalanceRSGroupCalled = false;
|
||||
boolean postBalanceRSGroupCalled = false;
|
||||
|
|
|
|||
|
|
@ -27,20 +27,19 @@ import java.util.ArrayList;
|
|||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HColumnDescriptor;
|
||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||
import org.apache.hadoop.hbase.NamespaceDescriptor;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.Waiter;
|
||||
import org.apache.hadoop.hbase.client.ClusterConnection;
|
||||
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptor;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.net.Address;
|
||||
import org.apache.hadoop.hbase.quotas.QuotaTableUtil;
|
||||
import org.apache.hadoop.hbase.quotas.QuotaUtil;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Assert;
|
||||
|
|
@ -52,10 +51,6 @@ import org.junit.experimental.categories.Category;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
|
||||
|
||||
@Category({ MediumTests.class })
|
||||
|
|
@ -126,8 +121,8 @@ public class TestRSGroupsBasics extends TestRSGroupsBase {
|
|||
RSGroupInfo appInfo = addGroup("appInfo", 1);
|
||||
admin.createNamespace(NamespaceDescriptor.create(nsName)
|
||||
.addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "appInfo").build());
|
||||
final HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||
desc.addFamily(new HColumnDescriptor("f"));
|
||||
final TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
|
||||
.setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
|
||||
admin.createTable(desc);
|
||||
// wait for created table to be assigned
|
||||
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
|
||||
|
|
@ -136,12 +131,9 @@ public class TestRSGroupsBasics extends TestRSGroupsBase {
|
|||
return getTableRegionMap().get(desc.getTableName()) != null;
|
||||
}
|
||||
});
|
||||
ServerName targetServer =
|
||||
ServerName.parseServerName(appInfo.getServers().iterator().next().toString());
|
||||
AdminProtos.AdminService.BlockingInterface rs =
|
||||
((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
|
||||
ServerName targetServer = getServerName(appInfo.getServers().iterator().next());
|
||||
// verify it was assigned to the right group
|
||||
Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(rs).size());
|
||||
Assert.assertEquals(1, admin.getRegions(targetServer).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -152,12 +144,7 @@ public class TestRSGroupsBasics extends TestRSGroupsBase {
|
|||
final RSGroupInfo appInfo = addGroup("appInfo", 1);
|
||||
Iterator<Address> iterator = appInfo.getServers().iterator();
|
||||
List<ServerName> serversToDecommission = new ArrayList<>();
|
||||
ServerName targetServer = ServerName.parseServerName(iterator.next().toString());
|
||||
AdminProtos.AdminService.BlockingInterface targetRS =
|
||||
((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
|
||||
targetServer = ProtobufUtil.toServerName(
|
||||
targetRS.getServerInfo(null, GetServerInfoRequest.newBuilder().build()).getServerInfo()
|
||||
.getServerName());
|
||||
ServerName targetServer = getServerName(iterator.next());
|
||||
assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer));
|
||||
serversToDecommission.add(targetServer);
|
||||
admin.decommissionRegionServers(serversToDecommission, true);
|
||||
|
|
@ -166,8 +153,8 @@ public class TestRSGroupsBasics extends TestRSGroupsBase {
|
|||
final TableName tableName = TableName.valueOf(tablePrefix + "_ns", name.getMethodName());
|
||||
admin.createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString())
|
||||
.addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build());
|
||||
final HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||
desc.addFamily(new HColumnDescriptor("f"));
|
||||
final TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
|
||||
.setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
|
||||
try {
|
||||
admin.createTable(desc);
|
||||
fail("Shouldn't create table successfully!");
|
||||
|
|
@ -181,7 +168,8 @@ public class TestRSGroupsBasics extends TestRSGroupsBase {
|
|||
admin.createTable(desc);
|
||||
// wait for created table to be assigned
|
||||
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
|
||||
@Override public boolean evaluate() throws Exception {
|
||||
@Override
|
||||
public boolean evaluate() throws Exception {
|
||||
return getTableRegionMap().get(desc.getTableName()) != null;
|
||||
}
|
||||
});
|
||||
|
|
@ -193,8 +181,8 @@ public class TestRSGroupsBasics extends TestRSGroupsBase {
|
|||
String tableName = tablePrefix + "_testCreateAndAssign";
|
||||
admin.modifyNamespace(NamespaceDescriptor.create("default")
|
||||
.addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "default").build());
|
||||
final HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
|
||||
desc.addFamily(new HColumnDescriptor("f"));
|
||||
final TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName))
|
||||
.setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
|
||||
admin.createTable(desc);
|
||||
// wait for created table to be assigned
|
||||
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
|
||||
|
|
@ -227,17 +215,11 @@ public class TestRSGroupsBasics extends TestRSGroupsBase {
|
|||
final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 3);
|
||||
NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size();
|
||||
|
||||
ServerName targetServer = ServerName.parseServerName(
|
||||
newGroup.getServers().iterator().next().toString());
|
||||
AdminProtos.AdminService.BlockingInterface targetRS =
|
||||
((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
|
||||
ServerName targetServer = getServerName(newGroup.getServers().iterator().next());
|
||||
try {
|
||||
targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null,
|
||||
GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName());
|
||||
// stopping may cause an exception
|
||||
// due to the connection loss
|
||||
targetRS.stopServer(null,
|
||||
AdminProtos.StopServerRequest.newBuilder().setReason("Die").build());
|
||||
admin.stopRegionServer(targetServer.getAddress().toString());
|
||||
NUM_DEAD_SERVERS++;
|
||||
} catch (Exception e) {
|
||||
}
|
||||
|
|
@ -245,8 +227,8 @@ public class TestRSGroupsBasics extends TestRSGroupsBase {
|
|||
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
|
||||
@Override
|
||||
public boolean evaluate() throws Exception {
|
||||
return !master.getServerManager().areDeadServersInProgress()
|
||||
&& cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS;
|
||||
return cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS &&
|
||||
!master.getServerManager().areDeadServersInProgress();
|
||||
}
|
||||
});
|
||||
assertFalse(cluster.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer));
|
||||
|
|
@ -267,17 +249,11 @@ public class TestRSGroupsBasics extends TestRSGroupsBase {
|
|||
LOG.info("testClearNotProcessedDeadServer");
|
||||
NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size();
|
||||
RSGroupInfo appInfo = addGroup("deadServerGroup", 1);
|
||||
ServerName targetServer =
|
||||
ServerName.parseServerName(appInfo.getServers().iterator().next().toString());
|
||||
AdminProtos.AdminService.BlockingInterface targetRS =
|
||||
((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
|
||||
ServerName targetServer = getServerName(appInfo.getServers().iterator().next());
|
||||
try {
|
||||
targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null,
|
||||
AdminProtos.GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName());
|
||||
// stopping may cause an exception
|
||||
// due to the connection loss
|
||||
targetRS.stopServer(null,
|
||||
AdminProtos.StopServerRequest.newBuilder().setReason("Die").build());
|
||||
admin.stopRegionServer(targetServer.getAddress().toString());
|
||||
NUM_DEAD_SERVERS++;
|
||||
} catch (Exception e) {
|
||||
}
|
||||
|
|
@ -287,8 +263,7 @@ public class TestRSGroupsBasics extends TestRSGroupsBase {
|
|||
return cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS;
|
||||
}
|
||||
});
|
||||
List<ServerName> notClearedServers =
|
||||
admin.clearDeadServers(Lists.newArrayList(targetServer));
|
||||
List<ServerName> notClearedServers = admin.clearDeadServers(Lists.newArrayList(targetServer));
|
||||
assertEquals(1, notClearedServers.size());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,21 +17,17 @@
|
|||
*/
|
||||
package org.apache.hadoop.hbase.rsgroup;
|
||||
|
||||
import static org.junit.Assert.assertFalse;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HColumnDescriptor;
|
||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||
import org.apache.hadoop.hbase.NamespaceDescriptor;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.Waiter;
|
||||
import org.apache.hadoop.hbase.client.ClusterConnection;
|
||||
import org.apache.hadoop.hbase.client.RegionInfo;
|
||||
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptor;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.net.Address;
|
||||
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Assert;
|
||||
|
|
@ -43,9 +39,6 @@ import org.junit.experimental.categories.Category;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
|
||||
|
||||
@Category({ MediumTests.class })
|
||||
|
|
@ -74,7 +67,7 @@ public class TestRSGroupsKillRS extends TestRSGroupsBase {
|
|||
|
||||
@After
|
||||
public void afterMethod() throws Exception {
|
||||
tearDownAfterMethod();
|
||||
// tearDownAfterMethod();
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -82,11 +75,10 @@ public class TestRSGroupsKillRS extends TestRSGroupsBase {
|
|||
RSGroupInfo appInfo = addGroup("appInfo", 1);
|
||||
|
||||
final TableName tableName = TableName.valueOf(tablePrefix + "_ns", name.getMethodName());
|
||||
admin.createNamespace(
|
||||
NamespaceDescriptor.create(tableName.getNamespaceAsString())
|
||||
admin.createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString())
|
||||
.addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build());
|
||||
final HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||
desc.addFamily(new HColumnDescriptor("f"));
|
||||
final TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
|
||||
.setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
|
||||
admin.createTable(desc);
|
||||
// wait for created table to be assigned
|
||||
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
|
||||
|
|
@ -96,32 +88,33 @@ public class TestRSGroupsKillRS extends TestRSGroupsBase {
|
|||
}
|
||||
});
|
||||
|
||||
ServerName targetServer = ServerName.parseServerName(
|
||||
appInfo.getServers().iterator().next().toString());
|
||||
AdminProtos.AdminService.BlockingInterface targetRS =
|
||||
((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
|
||||
RegionInfo targetRegion = ProtobufUtil.getOnlineRegions(targetRS).get(0);
|
||||
Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(targetRS).size());
|
||||
ServerName targetServer = getServerName(appInfo.getServers().iterator().next());
|
||||
Assert.assertEquals(1, admin.getRegions(targetServer).size());
|
||||
|
||||
try {
|
||||
// stopping may cause an exception
|
||||
// due to the connection loss
|
||||
targetRS.stopServer(null,
|
||||
AdminProtos.StopServerRequest.newBuilder().setReason("Die").build());
|
||||
admin.stopRegionServer(targetServer.getAddress().toString());
|
||||
} catch (Exception e) {
|
||||
}
|
||||
assertFalse(cluster.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer));
|
||||
|
||||
//wait for created table to be assigned
|
||||
// wait until the server is actually down
|
||||
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
|
||||
@Override
|
||||
public boolean evaluate() throws Exception {
|
||||
return cluster.getClusterMetrics().getRegionStatesInTransition().isEmpty();
|
||||
return !cluster.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer);
|
||||
}
|
||||
});
|
||||
// there is only one rs in the group and we killed it, so the region can not be online, until
|
||||
// later we add new servers to it.
|
||||
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
|
||||
@Override
|
||||
public boolean evaluate() throws Exception {
|
||||
return !cluster.getClusterMetrics().getRegionStatesInTransition().isEmpty();
|
||||
}
|
||||
});
|
||||
Set<Address> newServers = Sets.newHashSet();
|
||||
newServers.add(
|
||||
rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().iterator().next());
|
||||
newServers
|
||||
.add(rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().iterator().next());
|
||||
rsGroupAdmin.moveServers(newServers, appInfo.getName());
|
||||
|
||||
// Make sure all the table's regions get reassigned
|
||||
|
|
@ -137,13 +130,8 @@ public class TestRSGroupsKillRS extends TestRSGroupsBase {
|
|||
}
|
||||
});
|
||||
|
||||
targetServer = ServerName.parseServerName(
|
||||
newServers.iterator().next().toString());
|
||||
targetRS =
|
||||
((ClusterConnection) admin.getConnection()).getAdmin(targetServer);
|
||||
Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(targetRS).size());
|
||||
Assert.assertEquals(tableName,
|
||||
ProtobufUtil.getOnlineRegions(targetRS).get(0).getTable());
|
||||
ServerName targetServer1 = getServerName(newServers.iterator().next());
|
||||
Assert.assertEquals(1, admin.getRegions(targetServer1).size());
|
||||
Assert.assertEquals(tableName, admin.getRegions(targetServer1).get(0).getTable());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,19 +24,18 @@ import org.apache.hadoop.hbase.HBaseClassTestRule;
|
|||
import org.apache.hadoop.hbase.HBaseCluster;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.HRegionInfo;
|
||||
import org.apache.hadoop.hbase.MiniHBaseCluster;
|
||||
import org.apache.hadoop.hbase.StartMiniClusterOption;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.Waiter;
|
||||
import org.apache.hadoop.hbase.client.Admin;
|
||||
import org.apache.hadoop.hbase.client.RegionInfo;
|
||||
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
|
||||
import org.apache.hadoop.hbase.master.HMaster;
|
||||
import org.apache.hadoop.hbase.master.ServerManager;
|
||||
import org.apache.hadoop.hbase.regionserver.HRegionServer;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
|
|
@ -77,16 +76,13 @@ public class TestRSGroupsOfflineMode {
|
|||
@BeforeClass
|
||||
public static void setUp() throws Exception {
|
||||
TEST_UTIL = new HBaseTestingUtility();
|
||||
TEST_UTIL.getConfiguration().set(
|
||||
HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
|
||||
TEST_UTIL.getConfiguration().set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
|
||||
RSGroupBasedLoadBalancer.class.getName());
|
||||
TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
|
||||
RSGroupAdminEndpoint.class.getName());
|
||||
TEST_UTIL.getConfiguration().set(
|
||||
ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
|
||||
"1");
|
||||
StartMiniClusterOption option = StartMiniClusterOption.builder()
|
||||
.numMasters(2).numRegionServers(3).numDataNodes(3).build();
|
||||
TEST_UTIL.getConfiguration().set(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, "1");
|
||||
StartMiniClusterOption option =
|
||||
StartMiniClusterOption.builder().numMasters(2).numRegionServers(3).numDataNodes(3).build();
|
||||
TEST_UTIL.startMiniCluster(option);
|
||||
cluster = TEST_UTIL.getHBaseCluster();
|
||||
master = ((MiniHBaseCluster) cluster).getMaster();
|
||||
|
|
@ -121,15 +117,16 @@ public class TestRSGroupsOfflineMode {
|
|||
groupAdmin.addRSGroup(newGroup);
|
||||
if (master.getAssignmentManager().getRegionStates().getRegionAssignments()
|
||||
.containsValue(failoverRS.getServerName())) {
|
||||
for (HRegionInfo regionInfo : hbaseAdmin.getOnlineRegions(failoverRS.getServerName())) {
|
||||
for (RegionInfo regionInfo : hbaseAdmin.getRegions(failoverRS.getServerName())) {
|
||||
hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(),
|
||||
Bytes.toBytes(failoverRS.getServerName().getServerName()));
|
||||
}
|
||||
LOG.info("Waiting for region unassignments on failover RS...");
|
||||
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
|
||||
@Override public boolean evaluate() throws Exception {
|
||||
return !master.getServerManager().getLoad(failoverRS.getServerName())
|
||||
.getRegionMetrics().isEmpty();
|
||||
@Override
|
||||
public boolean evaluate() throws Exception {
|
||||
return !master.getServerManager().getLoad(failoverRS.getServerName()).getRegionMetrics()
|
||||
.isEmpty();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -163,19 +160,17 @@ public class TestRSGroupsOfflineMode {
|
|||
return TEST_UTIL.getHBaseCluster().getMaster() != null &&
|
||||
TEST_UTIL.getHBaseCluster().getMaster().isActiveMaster() &&
|
||||
TEST_UTIL.getHBaseCluster().getMaster().isInitialized() &&
|
||||
TEST_UTIL.getHBaseCluster().getMaster().getServerManager().getOnlineServers().size()
|
||||
<= 3;
|
||||
TEST_UTIL.getHBaseCluster().getMaster().getServerManager().getOnlineServers().size() <= 3;
|
||||
}
|
||||
});
|
||||
|
||||
// Get groupInfoManager from the new active master.
|
||||
RSGroupInfoManager groupMgr = ((MiniHBaseCluster)cluster).getMaster().getMasterCoprocessorHost()
|
||||
.findCoprocessor(RSGroupAdminEndpoint.class).getGroupInfoManager();
|
||||
RSGroupInfoManager groupMgr = ((MiniHBaseCluster) cluster).getMaster()
|
||||
.getMasterCoprocessorHost().findCoprocessor(RSGroupAdminEndpoint.class).getGroupInfoManager();
|
||||
// Make sure balancer is in offline mode, since this is what we're testing.
|
||||
assertFalse(groupMgr.isOnline());
|
||||
// Verify the group affiliation that's loaded from ZK instead of tables.
|
||||
assertEquals(newGroup,
|
||||
groupMgr.getRSGroupOfTable(RSGroupInfoManager.RSGROUP_TABLE_NAME));
|
||||
assertEquals(newGroup, groupMgr.getRSGroupOfTable(RSGroupInfoManager.RSGROUP_TABLE_NAME));
|
||||
assertEquals(RSGroupInfo.DEFAULT_GROUP, groupMgr.getRSGroupOfTable(failoverTable));
|
||||
// Kill final regionserver to see the failover happens for all tables except GROUP table since
|
||||
// it's group does not have any online RS.
|
||||
|
|
|
|||
Loading…
Reference in New Issue