From 5c88976fa3657356c854173589c88e3e7a887476 Mon Sep 17 00:00:00 2001 From: zhangduo Date: Sun, 13 Jan 2019 12:43:30 +0800 Subject: [PATCH] HBASE-21707 Fix warnings in hbase-rsgroup module and also make the UTs more stable Signed-off-by: Zheng Hu --- .../hbase/rsgroup/RSGroupAdminServer.java | 11 +- .../rsgroup/RSGroupBasedLoadBalancer.java | 4 +- .../balancer/RSGroupableBalancerTestBase.java | 19 +- .../TestRSGroupBasedLoadBalancer.java | 3 +- .../hbase/rsgroup/TestRSGroupsAdmin1.java | 184 +++++++------ .../hbase/rsgroup/TestRSGroupsAdmin2.java | 246 ++++++++---------- .../hbase/rsgroup/TestRSGroupsBalance.java | 71 +++-- .../hbase/rsgroup/TestRSGroupsBase.java | 22 +- .../hbase/rsgroup/TestRSGroupsBasics.java | 117 ++++----- .../hbase/rsgroup/TestRSGroupsKillRS.java | 86 +++--- .../rsgroup/TestRSGroupsOfflineMode.java | 67 +++-- 11 files changed, 381 insertions(+), 449 deletions(-) diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java index 9ac2d4d5972..e398e731ba1 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java @@ -27,7 +27,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ServerName; @@ -450,11 +449,11 @@ public class RSGroupAdminServer implements RSGroupAdmin { " servers; you must remove these servers from the RSGroup before" + "the RSGroup can be removed."); } - for (NamespaceDescriptor ns: master.getClusterSchema().getNamespaces()) { - String nsGroup = ns.getConfigurationValue(rsGroupInfo.NAMESPACE_DESC_PROP_GROUP); - if (nsGroup != null && nsGroup.equals(name)) { - throw new ConstraintException("RSGroup " + name + " is referenced by namespace: " + - ns.getName()); + for (NamespaceDescriptor ns : master.getClusterSchema().getNamespaces()) { + String nsGroup = ns.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); + if (nsGroup != null && nsGroup.equals(name)) { + throw new ConstraintException( + "RSGroup " + name + " is referenced by namespace: " + ns.getName()); } } rsGroupInfoManager.removeRSGroup(name); diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java index ecd034e76f1..cc14307e2f3 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java @@ -172,6 +172,7 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { ListMultimap regionMap = ArrayListMultimap.create(); ListMultimap serverMap = ArrayListMultimap.create(); generateGroupMaps(regions, servers, regionMap, serverMap); + LOG.info("=======" + regionMap + ", " + serverMap); for(String groupKey : regionMap.keySet()) { if (regionMap.get(groupKey).size() > 0) { Map> result = @@ -189,6 +190,7 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { } } } + LOG.info("=======" + assignments); return assignments; } @@ -296,7 +298,7 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer { return filterServers(RSGroupInfo.getServers(), onlineServers); } else { LOG.warn("RSGroup Information found to be null. Some regions might be unassigned."); - return Collections.EMPTY_LIST; + return Collections.emptyList(); } } diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java index 89b73e41c00..570bb3abb3e 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/RSGroupableBalancerTestBase.java @@ -31,14 +31,14 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; - import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionPlan; @@ -47,7 +47,6 @@ import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; import org.apache.hadoop.hbase.rsgroup.RSGroupInfoManager; import org.apache.hadoop.hbase.util.Bytes; - import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -71,7 +70,7 @@ public class RSGroupableBalancerTestBase { static List servers; static Map groupMap; static Map tableMap = new HashMap<>(); - static List tableDescs; + static List tableDescs; int[] regionAssignment = new int[] { 2, 5, 7, 10, 4, 3, 1 }; static int regionId = 0; @@ -390,19 +389,19 @@ public class RSGroupableBalancerTestBase { * @param hasBogusTable there is a table that does not determine the group * @return the list of table descriptors */ - protected static List constructTableDesc(boolean hasBogusTable) { - List tds = Lists.newArrayList(); + protected static List constructTableDesc(boolean hasBogusTable) { + List tds = Lists.newArrayList(); int index = rand.nextInt(groups.length); for (int i = 0; i < tables.length; i++) { - HTableDescriptor htd = new HTableDescriptor(tables[i]); - int grpIndex = (i + index) % groups.length ; + TableDescriptor htd = TableDescriptorBuilder.newBuilder(tables[i]).build(); + int grpIndex = (i + index) % groups.length; String groupName = groups[grpIndex]; tableMap.put(tables[i], groupName); tds.add(htd); } if (hasBogusTable) { tableMap.put(table0, ""); - tds.add(new HTableDescriptor(table0)); + tds.add(TableDescriptorBuilder.newBuilder(table0).build()); } return tds; } @@ -452,7 +451,7 @@ public class RSGroupableBalancerTestBase { } } - for(HTableDescriptor desc : tableDescs){ + for(TableDescriptor desc : tableDescs){ if(gm.getRSGroupOfTable(desc.getTableName()).endsWith(groupOfServer.getName())){ tableName = desc.getTableName(); } diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java index 091a08f055e..e39bbde228f 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/master/balancer/TestRSGroupBasedLoadBalancer.java @@ -109,8 +109,7 @@ public class TestRSGroupBasedLoadBalancer extends RSGroupableBalancerTestBase { .roundRobinAssignment(regions, servers); //test empty region/servers scenario //this should not throw an NPE - loadBalancer.roundRobinAssignment(regions, - Collections.EMPTY_LIST); + loadBalancer.roundRobinAssignment(regions, Collections.emptyList()); //test regular scenario assertTrue(assignments.keySet().size() == servers.size()); for (ServerName sn : assignments.keySet()) { diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java index a4a3a7cadd7..4d41d36962e 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin1.java @@ -28,23 +28,24 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.constraint.ConstraintException; import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.master.TableNamespaceManager; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.quotas.QuotaUtil; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; - import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -56,15 +57,14 @@ import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hbase.thirdparty.com.google.common.collect.Sets; -@Category({MediumTests.class}) +@Category({ MediumTests.class }) public class TestRSGroupsAdmin1 extends TestRSGroupsBase { @ClassRule public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRSGroupsAdmin1.class); + HBaseClassTestRule.forClass(TestRSGroupsAdmin1.class); protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsAdmin1.class); @@ -90,19 +90,19 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase { @Test public void testValidGroupNames() throws IOException { - String[] badNames = {"foo*","foo@","-"}; - String[] goodNames = {"foo_123"}; + String[] badNames = { "foo*", "foo@", "-" }; + String[] goodNames = { "foo_123" }; - for(String entry: badNames) { + for (String entry : badNames) { try { rsGroupAdmin.addRSGroup(entry); - fail("Expected a constraint exception for: "+entry); - } catch(ConstraintException ex) { - //expected + fail("Expected a constraint exception for: " + entry); + } catch (ConstraintException ex) { + // expected } } - for(String entry: goodNames) { + for (String entry : goodNames) { rsGroupAdmin.addRSGroup(entry); } } @@ -110,75 +110,71 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase { @Test public void testBogusArgs() throws Exception { assertNull(rsGroupAdmin.getRSGroupInfoOfTable(TableName.valueOf("nonexistent"))); - assertNull(rsGroupAdmin.getRSGroupOfServer(Address.fromParts("bogus",123))); + assertNull(rsGroupAdmin.getRSGroupOfServer(Address.fromParts("bogus", 123))); assertNull(rsGroupAdmin.getRSGroupInfo("bogus")); try { rsGroupAdmin.removeRSGroup("bogus"); fail("Expected removing bogus group to fail"); - } catch(ConstraintException ex) { - //expected + } catch (ConstraintException ex) { + // expected } try { rsGroupAdmin.moveTables(Sets.newHashSet(TableName.valueOf("bogustable")), "bogus"); fail("Expected move with bogus group to fail"); - } catch(ConstraintException|TableNotFoundException ex) { - //expected + } catch (ConstraintException | TableNotFoundException ex) { + // expected } try { - rsGroupAdmin.moveServers(Sets.newHashSet(Address.fromParts("bogus",123)), "bogus"); + rsGroupAdmin.moveServers(Sets.newHashSet(Address.fromParts("bogus", 123)), "bogus"); fail("Expected move with bogus group to fail"); - } catch(ConstraintException ex) { - //expected + } catch (ConstraintException ex) { + // expected } try { - admin.setBalancerRunning(true,true); + admin.balancerSwitch(true, true); rsGroupAdmin.balanceRSGroup("bogus"); - admin.setBalancerRunning(false,true); + admin.balancerSwitch(false, true); fail("Expected move with bogus group to fail"); - } catch(ConstraintException ex) { - //expected + } catch (ConstraintException ex) { + // expected } } @Test public void testNamespaceConstraint() throws Exception { - String nsName = tablePrefix+"_foo"; - String groupName = tablePrefix+"_foo"; + String nsName = tablePrefix + "_foo"; + String groupName = tablePrefix + "_foo"; LOG.info("testNamespaceConstraint"); rsGroupAdmin.addRSGroup(groupName); assertTrue(observer.preAddRSGroupCalled); assertTrue(observer.postAddRSGroupCalled); admin.createNamespace(NamespaceDescriptor.create(nsName) - .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, groupName) - .build()); - //test removing a referenced group + .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, groupName).build()); + // test removing a referenced group try { rsGroupAdmin.removeRSGroup(groupName); fail("Expected a constraint exception"); } catch (IOException ex) { } - //test modify group - //changing with the same name is fine - admin.modifyNamespace( - NamespaceDescriptor.create(nsName) - .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, groupName) - .build()); - String anotherGroup = tablePrefix+"_anotherGroup"; + // test modify group + // changing with the same name is fine + admin.modifyNamespace(NamespaceDescriptor.create(nsName) + .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, groupName).build()); + String anotherGroup = tablePrefix + "_anotherGroup"; rsGroupAdmin.addRSGroup(anotherGroup); - //test add non-existent group + // test add non-existent group admin.deleteNamespace(nsName); rsGroupAdmin.removeRSGroup(groupName); assertTrue(observer.preRemoveRSGroupCalled); assertTrue(observer.postRemoveRSGroupCalled); try { admin.createNamespace(NamespaceDescriptor.create(nsName) - .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "foo") - .build()); + .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "foo").build()); fail("Expected a constraint exception"); } catch (IOException ex) { } @@ -202,24 +198,24 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase { TEST_UTIL.createTable(tableName, Bytes.toBytes("f")); rsGroupAdmin.moveTables(Sets.newHashSet(tableName), "bar"); RSGroupInfo barGroup = rsGroupAdmin.getRSGroupInfo("bar"); - //group is not empty therefore it should fail + // group is not empty therefore it should fail try { rsGroupAdmin.removeRSGroup(barGroup.getName()); fail("Expected remove group to fail"); - } catch(IOException e) { + } catch (IOException e) { } - //group cannot lose all it's servers therefore it should fail + // group cannot lose all it's servers therefore it should fail try { rsGroupAdmin.moveServers(barGroup.getServers(), RSGroupInfo.DEFAULT_GROUP); fail("Expected move servers to fail"); - } catch(IOException e) { + } catch (IOException e) { } rsGroupAdmin.moveTables(barGroup.getTables(), RSGroupInfo.DEFAULT_GROUP); try { rsGroupAdmin.removeRSGroup(barGroup.getName()); fail("Expected move servers to fail"); - } catch(IOException e) { + } catch (IOException e) { } rsGroupAdmin.moveServers(barGroup.getServers(), RSGroupInfo.DEFAULT_GROUP); @@ -249,8 +245,8 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase { if (regionsB == null) { return false; } - return getTableRegionMap().get(tableNameA).size() >= 1 - && getTableRegionMap().get(tableNameB).size() >= 1; + return getTableRegionMap().get(tableNameA).size() >= 1 && + getTableRegionMap().get(tableNameB).size() >= 1; } }); @@ -259,24 +255,24 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase { RSGroupInfo tableGrpB = rsGroupAdmin.getRSGroupInfoOfTable(tableNameB); assertTrue(tableGrpB.getName().equals(RSGroupInfo.DEFAULT_GROUP)); - //change table's group + // change table's group LOG.info("Moving table [" + tableNameA + "," + tableNameB + "] to " + newGroup.getName()); rsGroupAdmin.moveTables(Sets.newHashSet(tableNameA, tableNameB), newGroup.getName()); - //verify group change + // verify group change Assert.assertEquals(newGroup.getName(), - rsGroupAdmin.getRSGroupInfoOfTable(tableNameA).getName()); + rsGroupAdmin.getRSGroupInfoOfTable(tableNameA).getName()); Assert.assertEquals(newGroup.getName(), - rsGroupAdmin.getRSGroupInfoOfTable(tableNameB).getName()); + rsGroupAdmin.getRSGroupInfoOfTable(tableNameB).getName()); - //verify tables' not exist in old group + // verify tables' not exist in old group Set DefaultTables = - rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables(); + rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables(); assertFalse(DefaultTables.contains(tableNameA)); assertFalse(DefaultTables.contains(tableNameB)); - //verify tables' exist in new group + // verify tables' exist in new group Set newGroupTables = rsGroupAdmin.getRSGroupInfo(newGroupName).getTables(); assertTrue(newGroupTables.contains(tableNameA)); assertTrue(newGroupTables.contains(tableNameB)); @@ -304,13 +300,13 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase { RSGroupInfo tableGrp = rsGroupAdmin.getRSGroupInfoOfTable(tableName); assertTrue(tableGrp.getName().equals(RSGroupInfo.DEFAULT_GROUP)); - //change table's group - LOG.info("Moving table "+tableName+" to "+newGroup.getName()); + // change table's group + LOG.info("Moving table " + tableName + " to " + newGroup.getName()); rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName()); - //verify group change + // verify group change Assert.assertEquals(newGroup.getName(), - rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName()); + rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName()); TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override @@ -328,14 +324,14 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase { } }); - //test truncate + // test truncate admin.disableTable(tableName); admin.truncateTable(tableName, true); Assert.assertEquals(1, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size()); - Assert.assertEquals(tableName, rsGroupAdmin.getRSGroupInfo( - newGroup.getName()).getTables().first()); + Assert.assertEquals(tableName, + rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().first()); - //verify removed table is removed from group + // verify removed table is removed from group TEST_UTIL.deleteTable(tableName); Assert.assertEquals(0, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size()); @@ -364,16 +360,16 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase { RSGroupInfo tableGrp = rsGroupAdmin.getRSGroupInfoOfTable(tableName); assertTrue(tableGrp.getName().equals(RSGroupInfo.DEFAULT_GROUP)); - //test disable table + // test disable table admin.disableTable(tableName); - //change table's group - LOG.info("Moving table "+ tableName + " to " + newGroup.getName()); + // change table's group + LOG.info("Moving table " + tableName + " to " + newGroup.getName()); rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName()); - //verify group change + // verify group change Assert.assertEquals(newGroup.getName(), - rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName()); + rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName()); } @Test @@ -383,27 +379,26 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase { RSGroupInfo tableGrp = rsGroupAdmin.getRSGroupInfoOfTable(tableName); assertNull(tableGrp); - //test if table exists already. + // test if table exists already. boolean exist = admin.tableExists(tableName); assertFalse(exist); - LOG.info("Moving table "+ tableName + " to " + RSGroupInfo.DEFAULT_GROUP); + LOG.info("Moving table " + tableName + " to " + RSGroupInfo.DEFAULT_GROUP); try { rsGroupAdmin.moveTables(Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP); fail("Table " + tableName + " shouldn't have been successfully moved."); - } catch(IOException ex) { + } catch (IOException ex) { assertTrue(ex instanceof TableNotFoundException); } try { - rsGroupAdmin.moveServersAndTables( - Sets.newHashSet(Address.fromParts("bogus",123)), - Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP); + rsGroupAdmin.moveServersAndTables(Sets.newHashSet(Address.fromParts("bogus", 123)), + Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP); fail("Table " + tableName + " shouldn't have been successfully moved."); - } catch(IOException ex) { + } catch (IOException ex) { assertTrue(ex instanceof TableNotFoundException); } - //verify group change + // verify group change assertNull(rsGroupAdmin.getRSGroupInfoOfTable(tableName)); } @@ -412,41 +407,39 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase { toggleQuotaCheckAndRestartMiniCluster(true); String nsp = "np1"; NamespaceDescriptor nspDesc = - NamespaceDescriptor.create(nsp) - .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5") - .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build(); + NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "5") + .addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2").build(); admin.createNamespace(nspDesc); assertEquals(3, admin.listNamespaceDescriptors().length); - HColumnDescriptor fam1 = new HColumnDescriptor("fam1"); - HTableDescriptor tableDescOne = - new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1")); - tableDescOne.addFamily(fam1); + ColumnFamilyDescriptor fam1 = ColumnFamilyDescriptorBuilder.of("fam1"); + TableDescriptor tableDescOne = TableDescriptorBuilder + .newBuilder(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1")) + .setColumnFamily(fam1).build(); admin.createTable(tableDescOne); - HTableDescriptor tableDescTwo = - new HTableDescriptor(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2")); - tableDescTwo.addFamily(fam1); + TableDescriptor tableDescTwo = TableDescriptorBuilder + .newBuilder(TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2")) + .setColumnFamily(fam1).build(); boolean constraintViolated = false; try { - admin.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), - 6); + admin.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 6); Assert.fail("Creation table should fail because of quota violation."); } catch (Exception exp) { assertTrue(exp instanceof IOException); constraintViolated = true; } finally { assertTrue("Constraint not violated for table " + tableDescTwo.getTableName(), - constraintViolated); + constraintViolated); } List rsGroupInfoList = rsGroupAdmin.listRSGroups(); boolean foundTable2 = false; boolean foundTable1 = false; - for(int i = 0; i < rsGroupInfoList.size(); i++){ - if(rsGroupInfoList.get(i).getTables().contains(tableDescTwo.getTableName())){ + for (int i = 0; i < rsGroupInfoList.size(); i++) { + if (rsGroupInfoList.get(i).getTables().contains(tableDescTwo.getTableName())) { foundTable2 = true; } - if(rsGroupInfoList.get(i).getTables().contains(tableDescOne.getTableName())){ + if (rsGroupInfoList.get(i).getTables().contains(tableDescOne.getTableName())) { foundTable1 = true; } } @@ -459,13 +452,12 @@ public class TestRSGroupsAdmin1 extends TestRSGroupsBase { } - private void toggleQuotaCheckAndRestartMiniCluster(boolean enable) throws Exception{ + private void toggleQuotaCheckAndRestartMiniCluster(boolean enable) throws Exception { TEST_UTIL.shutdownMiniCluster(); TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, enable); TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1); - TEST_UTIL.getConfiguration().setInt( - ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, - NUM_SLAVES_BASE - 1); + TEST_UTIL.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, + NUM_SLAVES_BASE - 1); TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true); initialize(); } diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java index f64e507c6f4..ccdbbc0ad94 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsAdmin2.java @@ -29,18 +29,16 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; - import org.apache.hadoop.hbase.ClusterMetrics.Option; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; -import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.constraint.ConstraintException; import org.apache.hadoop.hbase.net.Address; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; - import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -52,18 +50,14 @@ import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hbase.thirdparty.com.google.common.collect.Sets; -@Category({MediumTests.class}) +@Category({ MediumTests.class }) public class TestRSGroupsAdmin2 extends TestRSGroupsBase { @ClassRule public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRSGroupsAdmin2.class); + HBaseClassTestRule.forClass(TestRSGroupsAdmin2.class); protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsAdmin2.class); @@ -105,55 +99,50 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase { } }); - //get target region to move - Map> assignMap = - getTableServerRegionMap().get(tableName); + // get target region to move + Map> assignMap = getTableServerRegionMap().get(tableName); String targetRegion = null; - for(ServerName server : assignMap.keySet()) { + for (ServerName server : assignMap.keySet()) { targetRegion = assignMap.get(server).size() > 0 ? assignMap.get(server).get(0) : null; - if(targetRegion != null) { + if (targetRegion != null) { break; } } - //get server which is not a member of new group - ServerName targetServer = null; + // get server which is not a member of new group + ServerName tmpTargetServer = null; for (ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)) - .getLiveServerMetrics().keySet()) { + .getLiveServerMetrics().keySet()) { if (!newGroup.containsServer(server.getAddress())) { - targetServer = server; + tmpTargetServer = server; break; } } - - final AdminProtos.AdminService.BlockingInterface targetRS = - ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); - - //move target server to group - rsGroupAdmin.moveServers(Sets.newHashSet(targetServer.getAddress()), - newGroup.getName()); + final ServerName targetServer = tmpTargetServer; + // move target server to group + rsGroupAdmin.moveServers(Sets.newHashSet(targetServer.getAddress()), newGroup.getName()); TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { - return ProtobufUtil.getOnlineRegions(targetRS).size() <= 0; + return admin.getRegions(targetServer).size() <= 0; } }); // Lets move this region to the new group. - TEST_UTIL.getAdmin().move(Bytes.toBytes(RegionInfo.encodeRegionName( - Bytes.toBytes(targetRegion))), Bytes.toBytes(targetServer.getServerName())); + TEST_UTIL.getAdmin().move( + Bytes.toBytes(RegionInfo.encodeRegionName(Bytes.toBytes(targetRegion))), + Bytes.toBytes(targetServer.getServerName())); TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { - return - getTableRegionMap().get(tableName) != null && - getTableRegionMap().get(tableName).size() == 6 && - admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION)) - .getRegionStatesInTransition().size() < 1; + return getTableRegionMap().get(tableName) != null && + getTableRegionMap().get(tableName).size() == 6 && + admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION)) + .getRegionStatesInTransition().size() < 1; } }); - //verify that targetServer didn't open it - for (RegionInfo region: ProtobufUtil.getOnlineRegions(targetRS)) { + // verify that targetServer didn't open it + for (RegionInfo region : admin.getRegions(targetServer)) { if (targetRegion.equals(region.getRegionNameAsString())) { fail("Target server opened region"); } @@ -161,8 +150,7 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase { } @Test - public void testRegionServerMove() throws IOException, - InterruptedException { + public void testRegionServerMove() throws IOException, InterruptedException { int initNumGroups = rsGroupAdmin.listRSGroups().size(); RSGroupInfo appInfo = addGroup(getGroupName(name.getMethodName()), 1); RSGroupInfo adminInfo = addGroup(getGroupName(name.getMethodName()), 1); @@ -171,18 +159,16 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase { assertEquals(1, adminInfo.getServers().size()); assertEquals(1, appInfo.getServers().size()); assertEquals(getNumServers() - 2, dInfo.getServers().size()); - rsGroupAdmin.moveServers(appInfo.getServers(), - RSGroupInfo.DEFAULT_GROUP); + rsGroupAdmin.moveServers(appInfo.getServers(), RSGroupInfo.DEFAULT_GROUP); rsGroupAdmin.removeRSGroup(appInfo.getName()); - rsGroupAdmin.moveServers(adminInfo.getServers(), - RSGroupInfo.DEFAULT_GROUP); + rsGroupAdmin.moveServers(adminInfo.getServers(), RSGroupInfo.DEFAULT_GROUP); rsGroupAdmin.removeRSGroup(adminInfo.getName()); Assert.assertEquals(rsGroupAdmin.listRSGroups().size(), initNumGroups); } @Test public void testMoveServers() throws Exception { - //create groups and assign servers + // create groups and assign servers addGroup("bar", 3); rsGroupAdmin.addRSGroup("foo"); @@ -191,44 +177,44 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase { assertEquals(3, barGroup.getServers().size()); assertEquals(0, fooGroup.getServers().size()); - //test fail bogus server move + // test fail bogus server move try { - rsGroupAdmin.moveServers(Sets.newHashSet(Address.fromString("foo:9999")),"foo"); + rsGroupAdmin.moveServers(Sets.newHashSet(Address.fromString("foo:9999")), "foo"); fail("Bogus servers shouldn't have been successfully moved."); - } catch(IOException ex) { + } catch (IOException ex) { String exp = "Source RSGroup for server foo:9999 does not exist."; - String msg = "Expected '"+exp+"' in exception message: "; - assertTrue(msg+" "+ex.getMessage(), ex.getMessage().contains(exp)); + String msg = "Expected '" + exp + "' in exception message: "; + assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp)); } - //test success case - LOG.info("moving servers "+barGroup.getServers()+" to group foo"); + // test success case + LOG.info("moving servers " + barGroup.getServers() + " to group foo"); rsGroupAdmin.moveServers(barGroup.getServers(), fooGroup.getName()); barGroup = rsGroupAdmin.getRSGroupInfo("bar"); fooGroup = rsGroupAdmin.getRSGroupInfo("foo"); - assertEquals(0,barGroup.getServers().size()); - assertEquals(3,fooGroup.getServers().size()); + assertEquals(0, barGroup.getServers().size()); + assertEquals(3, fooGroup.getServers().size()); - LOG.info("moving servers "+fooGroup.getServers()+" to group default"); + LOG.info("moving servers " + fooGroup.getServers() + " to group default"); rsGroupAdmin.moveServers(fooGroup.getServers(), RSGroupInfo.DEFAULT_GROUP); TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { - return getNumServers() == - rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size(); + return getNumServers() == rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP) + .getServers().size(); } }); fooGroup = rsGroupAdmin.getRSGroupInfo("foo"); - assertEquals(0,fooGroup.getServers().size()); + assertEquals(0, fooGroup.getServers().size()); - //test group removal - LOG.info("Remove group "+barGroup.getName()); + // test group removal + LOG.info("Remove group " + barGroup.getName()); rsGroupAdmin.removeRSGroup(barGroup.getName()); Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(barGroup.getName())); - LOG.info("Remove group "+fooGroup.getName()); + LOG.info("Remove group " + fooGroup.getName()); rsGroupAdmin.removeRSGroup(fooGroup.getName()); Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName())); } @@ -238,15 +224,15 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase { LOG.info("testRemoveServers"); final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 3); Iterator
iterator = newGroup.getServers().iterator(); - ServerName targetServer = ServerName.parseServerName(iterator.next().toString()); + ServerName targetServer = getServerName(iterator.next()); // remove online servers try { rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress())); fail("Online servers shouldn't have been successfully removed."); - } catch(IOException ex) { - String exp = "Server " + targetServer.getAddress() - + " is an online server, not allowed to remove."; + } catch (IOException ex) { + String exp = + "Server " + targetServer.getAddress() + " is an online server, not allowed to remove."; String msg = "Expected '" + exp + "' in exception message: "; assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp)); } @@ -254,35 +240,30 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase { // remove dead servers NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size(); - AdminProtos.AdminService.BlockingInterface targetRS = - ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); try { - targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null, - GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName()); - //stopping may cause an exception - //due to the connection loss + // stopping may cause an exception + // due to the connection loss LOG.info("stopping server " + targetServer.getServerName()); - targetRS.stopServer(null, - AdminProtos.StopServerRequest.newBuilder().setReason("Die").build()); - NUM_DEAD_SERVERS ++; - } catch(Exception e) { + admin.stopRegionServer(targetServer.getAddress().toString()); + NUM_DEAD_SERVERS++; + } catch (Exception e) { } - //wait for stopped regionserver to dead server list + // wait for stopped regionserver to dead server list TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { - return !master.getServerManager().areDeadServersInProgress() - && cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS; + return !master.getServerManager().areDeadServersInProgress() && + cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS; } }); try { rsGroupAdmin.removeServers(Sets.newHashSet(targetServer.getAddress())); fail("Dead servers shouldn't have been successfully removed."); - } catch(IOException ex) { - String exp = "Server " + targetServer.getAddress() + " is on the dead servers list," - + " Maybe it will come back again, not allowed to remove."; + } catch (IOException ex) { + String exp = "Server " + targetServer.getAddress() + " is on the dead servers list," + + " Maybe it will come back again, not allowed to remove."; String msg = "Expected '" + exp + "' in exception message: "; assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp)); } @@ -290,10 +271,7 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase { // remove decommissioned servers List serversToDecommission = new ArrayList<>(); - targetServer = ServerName.parseServerName(iterator.next().toString()); - targetRS = ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); - targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null, - GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName()); + targetServer = getServerName(iterator.next()); assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer)); serversToDecommission.add(targetServer); @@ -314,7 +292,7 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase { public void testMoveServersAndTables() throws Exception { LOG.info("testMoveServersAndTables"); final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1); - //create table + // create table final byte[] familyNameBytes = Bytes.toBytes("f"); TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5); TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @@ -329,12 +307,12 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase { } }); - //get server which is not a member of new group + // get server which is not a member of new group ServerName targetServer = null; - for(ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)) + for (ServerName server : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)) .getLiveServerMetrics().keySet()) { - if(!newGroup.containsServer(server.getAddress()) && - !rsGroupAdmin.getRSGroupInfo("master").containsServer(server.getAddress())) { + if (!newGroup.containsServer(server.getAddress()) && + !rsGroupAdmin.getRSGroupInfo("master").containsServer(server.getAddress())) { targetServer = server; break; } @@ -342,95 +320,93 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase { LOG.debug("Print group info : " + rsGroupAdmin.listRSGroups()); int oldDefaultGroupServerSize = - rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size(); + rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size(); int oldDefaultGroupTableSize = - rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size(); + rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size(); - //test fail bogus server move + // test fail bogus server move try { rsGroupAdmin.moveServersAndTables(Sets.newHashSet(Address.fromString("foo:9999")), - Sets.newHashSet(tableName), newGroup.getName()); + Sets.newHashSet(tableName), newGroup.getName()); fail("Bogus servers shouldn't have been successfully moved."); - } catch(IOException ex) { + } catch (IOException ex) { String exp = "Source RSGroup for server foo:9999 does not exist."; String msg = "Expected '" + exp + "' in exception message: "; assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp)); } - //test fail server move + // test fail server move try { rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()), - Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP); + Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP); fail("servers shouldn't have been successfully moved."); - } catch(IOException ex) { - String exp = "Target RSGroup " + RSGroupInfo.DEFAULT_GROUP + - " is same as source " + RSGroupInfo.DEFAULT_GROUP + " RSGroup."; + } catch (IOException ex) { + String exp = "Target RSGroup " + RSGroupInfo.DEFAULT_GROUP + " is same as source " + + RSGroupInfo.DEFAULT_GROUP + " RSGroup."; String msg = "Expected '" + exp + "' in exception message: "; assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp)); } - //verify default group info + // verify default group info Assert.assertEquals(oldDefaultGroupServerSize, - rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()); + rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size()); Assert.assertEquals(oldDefaultGroupTableSize, - rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size()); + rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size()); - //verify new group info - Assert.assertEquals(1, - rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers().size()); - Assert.assertEquals(0, - rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size()); + // verify new group info + Assert.assertEquals(1, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers().size()); + Assert.assertEquals(0, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size()); - //get all region to move targetServer + // get all region to move targetServer List regionList = getTableRegionMap().get(tableName); - for(String region : regionList) { + for (String region : regionList) { // Lets move this region to the targetServer TEST_UTIL.getAdmin().move(Bytes.toBytes(RegionInfo.encodeRegionName(Bytes.toBytes(region))), - Bytes.toBytes(targetServer.getServerName())); + Bytes.toBytes(targetServer.getServerName())); } TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { return getTableRegionMap().get(tableName) != null && - getTableRegionMap().get(tableName).size() == 5 && - getTableServerRegionMap().get(tableName).size() == 1 && - admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION)) - .getRegionStatesInTransition().size() < 1; + getTableRegionMap().get(tableName).size() == 5 && + getTableServerRegionMap().get(tableName).size() == 1 && + admin.getClusterMetrics(EnumSet.of(Option.REGIONS_IN_TRANSITION)) + .getRegionStatesInTransition().size() < 1; } }); - //verify that all region move to targetServer + // verify that all region move to targetServer Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size()); - //move targetServer and table to newGroup + // move targetServer and table to newGroup LOG.info("moving server and table to newGroup"); rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()), - Sets.newHashSet(tableName), newGroup.getName()); + Sets.newHashSet(tableName), newGroup.getName()); - //verify group change + // verify group change Assert.assertEquals(newGroup.getName(), - rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName()); + rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName()); - //verify servers' not exist in old group - Set
defaultServers = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP) - .getServers(); + // verify servers' not exist in old group + Set
defaultServers = + rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers(); assertFalse(defaultServers.contains(targetServer.getAddress())); - //verify servers' exist in new group + // verify servers' exist in new group Set
newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers(); assertTrue(newGroupServers.contains(targetServer.getAddress())); - //verify tables' not exist in old group - Set defaultTables = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP) - .getTables(); + // verify tables' not exist in old group + Set defaultTables = + rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables(); assertFalse(defaultTables.contains(tableName)); - //verify tables' exist in new group + // verify tables' exist in new group Set newGroupTables = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables(); assertTrue(newGroupTables.contains(tableName)); - //verify that all region still assgin on targetServer + // verify that all region still assgin on targetServer Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size()); assertTrue(observer.preMoveServersAndTables); @@ -439,27 +415,27 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase { @Test public void testMoveServersFromDefaultGroup() throws Exception { - //create groups and assign servers + // create groups and assign servers rsGroupAdmin.addRSGroup("foo"); RSGroupInfo fooGroup = rsGroupAdmin.getRSGroupInfo("foo"); assertEquals(0, fooGroup.getServers().size()); RSGroupInfo defaultGroup = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP); - //test remove all servers from default + // test remove all servers from default try { rsGroupAdmin.moveServers(defaultGroup.getServers(), fooGroup.getName()); fail(RSGroupAdminServer.KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE); } catch (ConstraintException ex) { - assertTrue(ex.getMessage().contains(RSGroupAdminServer - .KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE)); + assertTrue( + ex.getMessage().contains(RSGroupAdminServer.KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE)); } - //test success case, remove one server from default ,keep at least one server + // test success case, remove one server from default ,keep at least one server if (defaultGroup.getServers().size() > 1) { Address serverInDefaultGroup = defaultGroup.getServers().iterator().next(); LOG.info("moving server " + serverInDefaultGroup + " from group default to group " + - fooGroup.getName()); + fooGroup.getName()); rsGroupAdmin.moveServers(Sets.newHashSet(serverInDefaultGroup), fooGroup.getName()); } @@ -470,15 +446,15 @@ public class TestRSGroupsAdmin2 extends TestRSGroupsBase { TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { - return getNumServers() == - rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size(); + return getNumServers() == rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP) + .getServers().size(); } }); fooGroup = rsGroupAdmin.getRSGroupInfo("foo"); assertEquals(0, fooGroup.getServers().size()); - //test group removal + // test group removal LOG.info("Remove group " + fooGroup.getName()); rsGroupAdmin.removeRSGroup(fooGroup.getName()); Assert.assertEquals(null, rsGroupAdmin.getRSGroupInfo(fooGroup.getName())); diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java index 6cb6c81e7e0..53cefe3aee6 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBalance.java @@ -23,18 +23,18 @@ import static org.junit.Assert.assertTrue; import java.util.List; import java.util.Map; - import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.Waiter.Predicate; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; - import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -45,15 +45,14 @@ import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hbase.thirdparty.com.google.common.collect.Sets; -@Category({MediumTests.class}) +@Category({ MediumTests.class }) public class TestRSGroupsBalance extends TestRSGroupsBase { @ClassRule public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRSGroupsBalance.class); + HBaseClassTestRule.forClass(TestRSGroupsBalance.class); protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsBalance.class); @@ -81,17 +80,15 @@ public class TestRSGroupsBalance extends TestRSGroupsBase { public void testGroupBalance() throws Exception { LOG.info(name.getMethodName()); String newGroupName = getGroupName(name.getMethodName()); - final RSGroupInfo newGroup = addGroup(newGroupName, 3); + addGroup(newGroupName, 3); - final TableName tableName = TableName.valueOf(tablePrefix+"_ns", name.getMethodName()); - admin.createNamespace( - NamespaceDescriptor.create(tableName.getNamespaceAsString()) - .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, newGroupName).build()); - final byte[] familyNameBytes = Bytes.toBytes("f"); - final HTableDescriptor desc = new HTableDescriptor(tableName); - desc.addFamily(new HColumnDescriptor("f")); - byte [] startKey = Bytes.toBytes("aaaaa"); - byte [] endKey = Bytes.toBytes("zzzzz"); + final TableName tableName = TableName.valueOf(tablePrefix + "_ns", name.getMethodName()); + admin.createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString()) + .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, newGroupName).build()); + final TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build(); + byte[] startKey = Bytes.toBytes("aaaaa"); + byte[] endKey = Bytes.toBytes("zzzzz"); admin.createTable(desc, startKey, endKey, 6); TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override @@ -104,12 +101,11 @@ public class TestRSGroupsBalance extends TestRSGroupsBase { } }); - //make assignment uneven, move all regions to one server - Map> assignMap = - getTableServerRegionMap().get(tableName); + // make assignment uneven, move all regions to one server + Map> assignMap = getTableServerRegionMap().get(tableName); final ServerName first = assignMap.entrySet().iterator().next().getKey(); - for(RegionInfo region: admin.getTableRegions(tableName)) { - if(!assignMap.get(first).contains(region.getRegionNameAsString())) { + for (RegionInfo region : admin.getRegions(tableName)) { + if (!assignMap.get(first).contains(region.getRegionNameAsString())) { admin.move(region.getEncodedNameAsBytes(), Bytes.toBytes(first.getServerName())); } } @@ -128,18 +124,18 @@ public class TestRSGroupsBalance extends TestRSGroupsBase { } }); - //balance the other group and make sure it doesn't affect the new group - admin.setBalancerRunning(true,true); + // balance the other group and make sure it doesn't affect the new group + admin.balancerSwitch(true, true); rsGroupAdmin.balanceRSGroup(RSGroupInfo.DEFAULT_GROUP); assertEquals(6, getTableServerRegionMap().get(tableName).get(first).size()); - //disable balance, balancer will not be run and return false - admin.setBalancerRunning(false,true); + // disable balance, balancer will not be run and return false + admin.balancerSwitch(false, true); assertFalse(rsGroupAdmin.balanceRSGroup(newGroupName)); assertEquals(6, getTableServerRegionMap().get(tableName).get(first).size()); - //enable balance - admin.setBalancerRunning(true,true); + // enable balance + admin.balancerSwitch(true, true); rsGroupAdmin.balanceRSGroup(newGroupName); TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override @@ -152,25 +148,25 @@ public class TestRSGroupsBalance extends TestRSGroupsBase { return true; } }); - admin.setBalancerRunning(false,true); + admin.balancerSwitch(false, true); } @Test public void testMisplacedRegions() throws Exception { - final TableName tableName = TableName.valueOf(tablePrefix+"_testMisplacedRegions"); + final TableName tableName = TableName.valueOf(tablePrefix + "_testMisplacedRegions"); LOG.info("testMisplacedRegions"); final RSGroupInfo RSGroupInfo = addGroup("testMisplacedRegions", 1); - TEST_UTIL.createMultiRegionTable(tableName, new byte[]{'f'}, 15); + TEST_UTIL.createMultiRegionTable(tableName, new byte[] { 'f' }, 15); TEST_UTIL.waitUntilAllRegionsAssigned(tableName); - rsGroupAdminEndpoint.getGroupInfoManager() - .moveTables(Sets.newHashSet(tableName), RSGroupInfo.getName()); + rsGroupAdminEndpoint.getGroupInfoManager().moveTables(Sets.newHashSet(tableName), + RSGroupInfo.getName()); - admin.setBalancerRunning(true,true); + admin.balancerSwitch(true, true); assertTrue(rsGroupAdmin.balanceRSGroup(RSGroupInfo.getName())); - admin.setBalancerRunning(false,true); + admin.balancerSwitch(false, true); assertTrue(observer.preBalanceRSGroupCalled); assertTrue(observer.postBalanceRSGroupCalled); @@ -178,9 +174,8 @@ public class TestRSGroupsBalance extends TestRSGroupsBase { @Override public boolean evaluate() throws Exception { ServerName serverName = - ServerName.valueOf(RSGroupInfo.getServers().iterator().next().toString(), 1); - return admin.getConnection().getAdmin() - .getOnlineRegions(serverName).size() == 15; + ServerName.valueOf(RSGroupInfo.getServers().iterator().next().toString(), 1); + return admin.getConnection().getAdmin().getRegions(serverName).size() == 15; } }); } diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java index 2d22087a9b2..48a8d9c501a 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java @@ -29,6 +29,7 @@ import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.TreeMap; +import java.util.regex.Pattern; import org.apache.hadoop.hbase.ClusterMetrics; import org.apache.hadoop.hbase.ClusterMetrics.Option; import org.apache.hadoop.hbase.HBaseCluster; @@ -81,7 +82,7 @@ public abstract class TestRSGroupsBase { protected static RSGroupAdminEndpoint rsGroupAdminEndpoint; protected static CPMasterObserver observer; - public final static long WAIT_TIMEOUT = 60000*5; + public final static long WAIT_TIMEOUT = 60000; public final static int NUM_SLAVES_BASE = 4; //number of slaves for the smallest cluster public static int NUM_DEAD_SERVERS = 0; @@ -110,7 +111,7 @@ public abstract class TestRSGroupsBase { protected static void initialize() throws Exception { admin = TEST_UTIL.getAdmin(); cluster = TEST_UTIL.getHBaseCluster(); - master = ((MiniHBaseCluster)cluster).getMaster(); + master = TEST_UTIL.getMiniHBaseCluster().getMaster(); //wait for balancer to come online TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @@ -120,7 +121,7 @@ public abstract class TestRSGroupsBase { ((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline(); } }); - admin.setBalancerRunning(false,true); + admin.balancerSwitch(false, true); rsGroupAdmin = new VerifyingRSGroupAdminClient( new RSGroupAdminClient(TEST_UTIL.getConnection()), TEST_UTIL.getConfiguration()); MasterCoprocessorHost host = master.getMasterCoprocessorHost(); @@ -206,7 +207,8 @@ public abstract class TestRSGroupsBase { } public void deleteTableIfNecessary() throws IOException { - for (TableDescriptor desc : TEST_UTIL.getAdmin().listTables(tablePrefix+".*")) { + for (TableDescriptor desc : TEST_UTIL.getAdmin() + .listTableDescriptors(Pattern.compile(tablePrefix + ".*"))) { TEST_UTIL.deleteTable(desc.getTableName()); } } @@ -287,7 +289,17 @@ public abstract class TestRSGroupsBase { } public String getGroupName(String baseName) { - return groupPrefix+"_"+baseName+"_"+rand.nextInt(Integer.MAX_VALUE); + return groupPrefix + "_" + baseName + "_" + rand.nextInt(Integer.MAX_VALUE); + } + + /** + * The server name in group does not contain the start code, this method will find out the start + * code and construct the ServerName object. + */ + protected ServerName getServerName(Address addr) { + return TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream() + .map(t -> t.getRegionServer().getServerName()).filter(sn -> sn.getAddress().equals(addr)) + .findFirst().get(); } public static class CPMasterObserver implements MasterCoprocessor, MasterObserver { diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java index 7415ab57119..748176e8080 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBasics.java @@ -27,20 +27,19 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Set; - import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; -import org.apache.hadoop.hbase.client.ClusterConnection; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.quotas.QuotaTableUtil; import org.apache.hadoop.hbase.quotas.QuotaUtil; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; - import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -52,18 +51,14 @@ import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetServerInfoRequest; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; -@Category({MediumTests.class}) +@Category({ MediumTests.class }) public class TestRSGroupsBasics extends TestRSGroupsBase { @ClassRule public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRSGroupsBasics.class); + HBaseClassTestRule.forClass(TestRSGroupsBasics.class); protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsBasics.class); @@ -100,7 +95,7 @@ public class TestRSGroupsBasics extends TestRSGroupsBase { @Test public void testCreateAndDrop() throws Exception { TEST_UTIL.createTable(tableName, Bytes.toBytes("cf")); - //wait for created table to be assigned + // wait for created table to be assigned TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { @@ -112,36 +107,33 @@ public class TestRSGroupsBasics extends TestRSGroupsBase { @Test public void testCreateMultiRegion() throws IOException { - byte[] end = {1,3,5,7,9}; - byte[] start = {0,2,4,6,8}; - byte[][] f = {Bytes.toBytes("f")}; - TEST_UTIL.createTable(tableName, f,1,start,end,10); + byte[] end = { 1, 3, 5, 7, 9 }; + byte[] start = { 0, 2, 4, 6, 8 }; + byte[][] f = { Bytes.toBytes("f") }; + TEST_UTIL.createTable(tableName, f, 1, start, end, 10); } @Test public void testNamespaceCreateAndAssign() throws Exception { LOG.info("testNamespaceCreateAndAssign"); - String nsName = tablePrefix+"_foo"; + String nsName = tablePrefix + "_foo"; final TableName tableName = TableName.valueOf(nsName, tablePrefix + "_testCreateAndAssign"); RSGroupInfo appInfo = addGroup("appInfo", 1); admin.createNamespace(NamespaceDescriptor.create(nsName) - .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "appInfo").build()); - final HTableDescriptor desc = new HTableDescriptor(tableName); - desc.addFamily(new HColumnDescriptor("f")); + .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "appInfo").build()); + final TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build(); admin.createTable(desc); - //wait for created table to be assigned + // wait for created table to be assigned TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { return getTableRegionMap().get(desc.getTableName()) != null; } }); - ServerName targetServer = - ServerName.parseServerName(appInfo.getServers().iterator().next().toString()); - AdminProtos.AdminService.BlockingInterface rs = - ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); - //verify it was assigned to the right group - Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(rs).size()); + ServerName targetServer = getServerName(appInfo.getServers().iterator().next()); + // verify it was assigned to the right group + Assert.assertEquals(1, admin.getRegions(targetServer).size()); } @Test @@ -152,12 +144,7 @@ public class TestRSGroupsBasics extends TestRSGroupsBase { final RSGroupInfo appInfo = addGroup("appInfo", 1); Iterator
iterator = appInfo.getServers().iterator(); List serversToDecommission = new ArrayList<>(); - ServerName targetServer = ServerName.parseServerName(iterator.next().toString()); - AdminProtos.AdminService.BlockingInterface targetRS = - ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); - targetServer = ProtobufUtil.toServerName( - targetRS.getServerInfo(null, GetServerInfoRequest.newBuilder().build()).getServerInfo() - .getServerName()); + ServerName targetServer = getServerName(iterator.next()); assertTrue(master.getServerManager().getOnlineServers().containsKey(targetServer)); serversToDecommission.add(targetServer); admin.decommissionRegionServers(serversToDecommission, true); @@ -165,9 +152,9 @@ public class TestRSGroupsBasics extends TestRSGroupsBase { final TableName tableName = TableName.valueOf(tablePrefix + "_ns", name.getMethodName()); admin.createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString()) - .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build()); - final HTableDescriptor desc = new HTableDescriptor(tableName); - desc.addFamily(new HColumnDescriptor("f")); + .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build()); + final TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build(); try { admin.createTable(desc); fail("Shouldn't create table successfully!"); @@ -181,7 +168,8 @@ public class TestRSGroupsBasics extends TestRSGroupsBase { admin.createTable(desc); // wait for created table to be assigned TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { - @Override public boolean evaluate() throws Exception { + @Override + public boolean evaluate() throws Exception { return getTableRegionMap().get(desc.getTableName()) != null; } }); @@ -192,11 +180,11 @@ public class TestRSGroupsBasics extends TestRSGroupsBase { LOG.info("testDefaultNamespaceCreateAndAssign"); String tableName = tablePrefix + "_testCreateAndAssign"; admin.modifyNamespace(NamespaceDescriptor.create("default") - .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "default").build()); - final HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); - desc.addFamily(new HColumnDescriptor("f")); + .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "default").build()); + final TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName)) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build(); admin.createTable(desc); - //wait for created table to be assigned + // wait for created table to be assigned TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { @@ -227,33 +215,27 @@ public class TestRSGroupsBasics extends TestRSGroupsBase { final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 3); NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size(); - ServerName targetServer = ServerName.parseServerName( - newGroup.getServers().iterator().next().toString()); - AdminProtos.AdminService.BlockingInterface targetRS = - ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); + ServerName targetServer = getServerName(newGroup.getServers().iterator().next()); try { - targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null, - GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName()); - //stopping may cause an exception - //due to the connection loss - targetRS.stopServer(null, - AdminProtos.StopServerRequest.newBuilder().setReason("Die").build()); - NUM_DEAD_SERVERS ++; - } catch(Exception e) { + // stopping may cause an exception + // due to the connection loss + admin.stopRegionServer(targetServer.getAddress().toString()); + NUM_DEAD_SERVERS++; + } catch (Exception e) { } - //wait for stopped regionserver to dead server list + // wait for stopped regionserver to dead server list TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { - return !master.getServerManager().areDeadServersInProgress() - && cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS; + return cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS && + !master.getServerManager().areDeadServersInProgress(); } }); assertFalse(cluster.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer)); assertTrue(cluster.getClusterMetrics().getDeadServerNames().contains(targetServer)); assertTrue(newGroup.getServers().contains(targetServer.getAddress())); - //clear dead servers list + // clear dead servers list List notClearedServers = admin.clearDeadServers(Lists.newArrayList(targetServer)); assertEquals(0, notClearedServers.size()); @@ -267,19 +249,13 @@ public class TestRSGroupsBasics extends TestRSGroupsBase { LOG.info("testClearNotProcessedDeadServer"); NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size(); RSGroupInfo appInfo = addGroup("deadServerGroup", 1); - ServerName targetServer = - ServerName.parseServerName(appInfo.getServers().iterator().next().toString()); - AdminProtos.AdminService.BlockingInterface targetRS = - ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); + ServerName targetServer = getServerName(appInfo.getServers().iterator().next()); try { - targetServer = ProtobufUtil.toServerName(targetRS.getServerInfo(null, - AdminProtos.GetServerInfoRequest.newBuilder().build()).getServerInfo().getServerName()); - //stopping may cause an exception - //due to the connection loss - targetRS.stopServer(null, - AdminProtos.StopServerRequest.newBuilder().setReason("Die").build()); - NUM_DEAD_SERVERS ++; - } catch(Exception e) { + // stopping may cause an exception + // due to the connection loss + admin.stopRegionServer(targetServer.getAddress().toString()); + NUM_DEAD_SERVERS++; + } catch (Exception e) { } TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override @@ -287,8 +263,7 @@ public class TestRSGroupsBasics extends TestRSGroupsBase { return cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS; } }); - List notClearedServers = - admin.clearDeadServers(Lists.newArrayList(targetServer)); + List notClearedServers = admin.clearDeadServers(Lists.newArrayList(targetServer)); assertEquals(1, notClearedServers.size()); } diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java index bce4cc9af0a..ec58140ffe0 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsKillRS.java @@ -17,21 +17,17 @@ */ package org.apache.hadoop.hbase.rsgroup; -import static org.junit.Assert.assertFalse; - import java.util.Set; - import org.apache.hadoop.hbase.HBaseClassTestRule; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; -import org.apache.hadoop.hbase.client.ClusterConnection; -import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.net.Address; - +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -43,17 +39,14 @@ import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hbase.thirdparty.com.google.common.collect.Sets; -@Category({MediumTests.class}) +@Category({ MediumTests.class }) public class TestRSGroupsKillRS extends TestRSGroupsBase { @ClassRule public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRSGroupsKillRS.class); + HBaseClassTestRule.forClass(TestRSGroupsKillRS.class); protected static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsKillRS.class); @@ -74,21 +67,20 @@ public class TestRSGroupsKillRS extends TestRSGroupsBase { @After public void afterMethod() throws Exception { - tearDownAfterMethod(); + // tearDownAfterMethod(); } @Test public void testKillRS() throws Exception { RSGroupInfo appInfo = addGroup("appInfo", 1); - final TableName tableName = TableName.valueOf(tablePrefix+"_ns", name.getMethodName()); - admin.createNamespace( - NamespaceDescriptor.create(tableName.getNamespaceAsString()) - .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build()); - final HTableDescriptor desc = new HTableDescriptor(tableName); - desc.addFamily(new HColumnDescriptor("f")); + final TableName tableName = TableName.valueOf(tablePrefix + "_ns", name.getMethodName()); + admin.createNamespace(NamespaceDescriptor.create(tableName.getNamespaceAsString()) + .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, appInfo.getName()).build()); + final TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build(); admin.createTable(desc); - //wait for created table to be assigned + // wait for created table to be assigned TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { @@ -96,40 +88,41 @@ public class TestRSGroupsKillRS extends TestRSGroupsBase { } }); - ServerName targetServer = ServerName.parseServerName( - appInfo.getServers().iterator().next().toString()); - AdminProtos.AdminService.BlockingInterface targetRS = - ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); - RegionInfo targetRegion = ProtobufUtil.getOnlineRegions(targetRS).get(0); - Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(targetRS).size()); + ServerName targetServer = getServerName(appInfo.getServers().iterator().next()); + Assert.assertEquals(1, admin.getRegions(targetServer).size()); try { - //stopping may cause an exception - //due to the connection loss - targetRS.stopServer(null, - AdminProtos.StopServerRequest.newBuilder().setReason("Die").build()); - } catch(Exception e) { + // stopping may cause an exception + // due to the connection loss + admin.stopRegionServer(targetServer.getAddress().toString()); + } catch (Exception e) { } - assertFalse(cluster.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer)); - - //wait for created table to be assigned + // wait until the server is actually down TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { - return cluster.getClusterMetrics().getRegionStatesInTransition().isEmpty(); + return !cluster.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer); + } + }); + // there is only one rs in the group and we killed it, so the region can not be online, until + // later we add new servers to it. + TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return !cluster.getClusterMetrics().getRegionStatesInTransition().isEmpty(); } }); Set
newServers = Sets.newHashSet(); - newServers.add( - rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().iterator().next()); + newServers + .add(rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().iterator().next()); rsGroupAdmin.moveServers(newServers, appInfo.getName()); - //Make sure all the table's regions get reassigned - //disabling the table guarantees no conflicting assign/unassign (ie SSH) happens + // Make sure all the table's regions get reassigned + // disabling the table guarantees no conflicting assign/unassign (ie SSH) happens admin.disableTable(tableName); admin.enableTable(tableName); - //wait for region to be assigned + // wait for region to be assigned TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { @@ -137,13 +130,8 @@ public class TestRSGroupsKillRS extends TestRSGroupsBase { } }); - targetServer = ServerName.parseServerName( - newServers.iterator().next().toString()); - targetRS = - ((ClusterConnection) admin.getConnection()).getAdmin(targetServer); - Assert.assertEquals(1, ProtobufUtil.getOnlineRegions(targetRS).size()); - Assert.assertEquals(tableName, - ProtobufUtil.getOnlineRegions(targetRS).get(0).getTable()); + ServerName targetServer1 = getServerName(newServers.iterator().next()); + Assert.assertEquals(1, admin.getRegions(targetServer1).size()); + Assert.assertEquals(tableName, admin.getRegions(targetServer1).get(0).getTable()); } - } diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java index b93de57bd8b..8058439a945 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsOfflineMode.java @@ -24,19 +24,18 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseCluster; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.StartMiniClusterOption; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; - import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -62,7 +61,7 @@ public class TestRSGroupsOfflineMode { @ClassRule public static final HBaseClassTestRule CLASS_RULE = - HBaseClassTestRule.forClass(TestRSGroupsOfflineMode.class); + HBaseClassTestRule.forClass(TestRSGroupsOfflineMode.class); private static final Logger LOG = LoggerFactory.getLogger(TestRSGroupsOfflineMode.class); private static HMaster master; @@ -77,28 +76,25 @@ public class TestRSGroupsOfflineMode { @BeforeClass public static void setUp() throws Exception { TEST_UTIL = new HBaseTestingUtility(); - TEST_UTIL.getConfiguration().set( - HConstants.HBASE_MASTER_LOADBALANCER_CLASS, - RSGroupBasedLoadBalancer.class.getName()); + TEST_UTIL.getConfiguration().set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + RSGroupBasedLoadBalancer.class.getName()); TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, - RSGroupAdminEndpoint.class.getName()); - TEST_UTIL.getConfiguration().set( - ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, - "1"); - StartMiniClusterOption option = StartMiniClusterOption.builder() - .numMasters(2).numRegionServers(3).numDataNodes(3).build(); + RSGroupAdminEndpoint.class.getName()); + TEST_UTIL.getConfiguration().set(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, "1"); + StartMiniClusterOption option = + StartMiniClusterOption.builder().numMasters(2).numRegionServers(3).numDataNodes(3).build(); TEST_UTIL.startMiniCluster(option); cluster = TEST_UTIL.getHBaseCluster(); - master = ((MiniHBaseCluster)cluster).getMaster(); + master = ((MiniHBaseCluster) cluster).getMaster(); master.balanceSwitch(false); hbaseAdmin = TEST_UTIL.getAdmin(); - //wait till the balancer is in online mode + // wait till the balancer is in online mode TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { return master.isInitialized() && - ((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline() && - master.getServerManager().getOnlineServersList().size() >= 3; + ((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline() && + master.getServerManager().getOnlineServersList().size() >= 3; } }); } @@ -113,23 +109,24 @@ public class TestRSGroupsOfflineMode { // Table should be after group table name so it gets assigned later. final TableName failoverTable = TableName.valueOf(name.getMethodName()); TEST_UTIL.createTable(failoverTable, Bytes.toBytes("f")); - final HRegionServer killRS = ((MiniHBaseCluster)cluster).getRegionServer(0); - final HRegionServer groupRS = ((MiniHBaseCluster)cluster).getRegionServer(1); - final HRegionServer failoverRS = ((MiniHBaseCluster)cluster).getRegionServer(2); - String newGroup = "my_group"; + final HRegionServer killRS = ((MiniHBaseCluster) cluster).getRegionServer(0); + final HRegionServer groupRS = ((MiniHBaseCluster) cluster).getRegionServer(1); + final HRegionServer failoverRS = ((MiniHBaseCluster) cluster).getRegionServer(2); + String newGroup = "my_group"; RSGroupAdmin groupAdmin = new RSGroupAdminClient(TEST_UTIL.getConnection()); groupAdmin.addRSGroup(newGroup); - if(master.getAssignmentManager().getRegionStates().getRegionAssignments() - .containsValue(failoverRS.getServerName())) { - for (HRegionInfo regionInfo : hbaseAdmin.getOnlineRegions(failoverRS.getServerName())) { + if (master.getAssignmentManager().getRegionStates().getRegionAssignments() + .containsValue(failoverRS.getServerName())) { + for (RegionInfo regionInfo : hbaseAdmin.getRegions(failoverRS.getServerName())) { hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(), - Bytes.toBytes(failoverRS.getServerName().getServerName())); + Bytes.toBytes(failoverRS.getServerName().getServerName())); } LOG.info("Waiting for region unassignments on failover RS..."); TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() { - @Override public boolean evaluate() throws Exception { - return !master.getServerManager().getLoad(failoverRS.getServerName()) - .getRegionMetrics().isEmpty(); + @Override + public boolean evaluate() throws Exception { + return !master.getServerManager().getLoad(failoverRS.getServerName()).getRegionMetrics() + .isEmpty(); } }); } @@ -140,7 +137,7 @@ public class TestRSGroupsOfflineMode { @Override public boolean evaluate() throws Exception { return groupRS.getNumberOfOnlineRegions() < 1 && - master.getAssignmentManager().getRegionStates().getRegionsInTransitionCount() < 1; + master.getAssignmentManager().getRegionStates().getRegionsInTransitionCount() < 1; } }); // Move table to group and wait. @@ -161,21 +158,19 @@ public class TestRSGroupsOfflineMode { @Override public boolean evaluate() throws Exception { return TEST_UTIL.getHBaseCluster().getMaster() != null && - TEST_UTIL.getHBaseCluster().getMaster().isActiveMaster() && - TEST_UTIL.getHBaseCluster().getMaster().isInitialized() && - TEST_UTIL.getHBaseCluster().getMaster().getServerManager().getOnlineServers().size() - <= 3; + TEST_UTIL.getHBaseCluster().getMaster().isActiveMaster() && + TEST_UTIL.getHBaseCluster().getMaster().isInitialized() && + TEST_UTIL.getHBaseCluster().getMaster().getServerManager().getOnlineServers().size() <= 3; } }); // Get groupInfoManager from the new active master. - RSGroupInfoManager groupMgr = ((MiniHBaseCluster)cluster).getMaster().getMasterCoprocessorHost() - .findCoprocessor(RSGroupAdminEndpoint.class).getGroupInfoManager(); + RSGroupInfoManager groupMgr = ((MiniHBaseCluster) cluster).getMaster() + .getMasterCoprocessorHost().findCoprocessor(RSGroupAdminEndpoint.class).getGroupInfoManager(); // Make sure balancer is in offline mode, since this is what we're testing. assertFalse(groupMgr.isOnline()); // Verify the group affiliation that's loaded from ZK instead of tables. - assertEquals(newGroup, - groupMgr.getRSGroupOfTable(RSGroupInfoManager.RSGROUP_TABLE_NAME)); + assertEquals(newGroup, groupMgr.getRSGroupOfTable(RSGroupInfoManager.RSGROUP_TABLE_NAME)); assertEquals(RSGroupInfo.DEFAULT_GROUP, groupMgr.getRSGroupOfTable(failoverTable)); // Kill final regionserver to see the failover happens for all tables except GROUP table since // it's group does not have any online RS.