diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java index c34fb48bb6d..5da9d034893 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java @@ -32,13 +32,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; -import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; @@ -58,7 +56,6 @@ import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper; import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer; import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan; import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan.Position; -import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -83,9 +80,6 @@ public class TestRegionPlacement { private int REGION_NUM = 10; private Map favoredNodesAssignmentPlan = new HashMap(); - private final static int PRIMARY = Position.PRIMARY.ordinal(); - private final static int SECONDARY = Position.SECONDARY.ordinal(); - private final static int TERTIARY = Position.TERTIARY.ordinal(); @BeforeClass public static void setupBeforeClass() throws Exception { @@ -104,111 +98,6 @@ public class TestRegionPlacement { TEST_UTIL.shutdownMiniCluster(); } - @Test - public void testFavoredNodesPresentForRoundRobinAssignment() throws HBaseIOException { - LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration()); - balancer.setMasterServices(TEST_UTIL.getMiniHBaseCluster().getMaster()); - List servers = new ArrayList(); - for (int i = 0; i < SLAVES; i++) { - ServerName server = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i).getServerName(); - servers.add(server); - } - List regions = new ArrayList(1); - HRegionInfo region = new HRegionInfo(TableName.valueOf("foobar")); - regions.add(region); - Map> assignmentMap = balancer.roundRobinAssignment(regions, - servers); - Set serverBefore = assignmentMap.keySet(); - List favoredNodesBefore = - ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region); - assertTrue(favoredNodesBefore.size() == 3); - // the primary RS should be the one that the balancer's assignment returns - assertTrue(ServerName.isSameHostnameAndPort(serverBefore.iterator().next(), - favoredNodesBefore.get(PRIMARY))); - // now remove the primary from the list of available servers - List removedServers = removeMatchingServers(serverBefore, servers); - // call roundRobinAssignment with the modified servers list - assignmentMap = balancer.roundRobinAssignment(regions, servers); - List favoredNodesAfter = - ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region); - assertTrue(favoredNodesAfter.size() == 3); - // We don't expect the favored nodes assignments to change in multiple calls - // to the roundRobinAssignment method in the balancer (relevant for AssignmentManager.assign - // failures) - assertTrue(favoredNodesAfter.containsAll(favoredNodesBefore)); - Set serverAfter = assignmentMap.keySet(); - // We expect the new RegionServer assignee to be one of the favored nodes - // chosen earlier. - assertTrue(ServerName.isSameHostnameAndPort(serverAfter.iterator().next(), - favoredNodesBefore.get(SECONDARY)) || - ServerName.isSameHostnameAndPort(serverAfter.iterator().next(), - favoredNodesBefore.get(TERTIARY))); - - // put back the primary in the list of available servers - servers.addAll(removedServers); - // now roundRobinAssignment with the modified servers list should return the primary - // as the regionserver assignee - assignmentMap = balancer.roundRobinAssignment(regions, servers); - Set serverWithPrimary = assignmentMap.keySet(); - assertTrue(serverBefore.containsAll(serverWithPrimary)); - - // Make all the favored nodes unavailable for assignment - removeMatchingServers(favoredNodesAfter, servers); - // call roundRobinAssignment with the modified servers list - assignmentMap = balancer.roundRobinAssignment(regions, servers); - List favoredNodesNow = - ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region); - assertTrue(favoredNodesNow.size() == 3); - assertTrue(!favoredNodesNow.contains(favoredNodesAfter.get(PRIMARY)) && - !favoredNodesNow.contains(favoredNodesAfter.get(SECONDARY)) && - !favoredNodesNow.contains(favoredNodesAfter.get(TERTIARY))); - } - - @Test - public void testFavoredNodesPresentForRandomAssignment() throws HBaseIOException { - LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration()); - balancer.setMasterServices(TEST_UTIL.getMiniHBaseCluster().getMaster()); - List servers = new ArrayList(); - for (int i = 0; i < SLAVES; i++) { - ServerName server = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i).getServerName(); - servers.add(server); - } - List regions = new ArrayList(1); - HRegionInfo region = new HRegionInfo(TableName.valueOf("foobar")); - regions.add(region); - ServerName serverBefore = balancer.randomAssignment(region, servers); - List favoredNodesBefore = - ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region); - assertTrue(favoredNodesBefore.size() == 3); - // the primary RS should be the one that the balancer's assignment returns - assertTrue(ServerName.isSameHostnameAndPort(serverBefore,favoredNodesBefore.get(PRIMARY))); - // now remove the primary from the list of servers - removeMatchingServers(serverBefore, servers); - // call randomAssignment with the modified servers list - ServerName serverAfter = balancer.randomAssignment(region, servers); - List favoredNodesAfter = - ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region); - assertTrue(favoredNodesAfter.size() == 3); - // We don't expect the favored nodes assignments to change in multiple calls - // to the randomAssignment method in the balancer (relevant for AssignmentManager.assign - // failures) - assertTrue(favoredNodesAfter.containsAll(favoredNodesBefore)); - // We expect the new RegionServer assignee to be one of the favored nodes - // chosen earlier. - assertTrue(ServerName.isSameHostnameAndPort(serverAfter, favoredNodesBefore.get(SECONDARY)) || - ServerName.isSameHostnameAndPort(serverAfter, favoredNodesBefore.get(TERTIARY))); - // Make all the favored nodes unavailable for assignment - removeMatchingServers(favoredNodesAfter, servers); - // call randomAssignment with the modified servers list - balancer.randomAssignment(region, servers); - List favoredNodesNow = - ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region); - assertTrue(favoredNodesNow.size() == 3); - assertTrue(!favoredNodesNow.contains(favoredNodesAfter.get(PRIMARY)) && - !favoredNodesNow.contains(favoredNodesAfter.get(SECONDARY)) && - !favoredNodesNow.contains(favoredNodesAfter.get(TERTIARY))); - } - @Test public void testRegionPlacement() throws Exception { String tableStr = "testRegionAssignment"; @@ -501,27 +390,6 @@ public class TestRegionPlacement { regionMovement, expected); } - private List removeMatchingServers(ServerName serverWithoutStartCode, - List servers) { - List serversToRemove = new ArrayList(); - for (ServerName s : servers) { - if (ServerName.isSameHostnameAndPort(s, serverWithoutStartCode)) { - serversToRemove.add(s); - } - } - servers.removeAll(serversToRemove); - return serversToRemove; - } - - private List removeMatchingServers(Collection serversWithoutStartCode, - List servers) { - List serversToRemove = new ArrayList(); - for (ServerName s : serversWithoutStartCode) { - serversToRemove.addAll(removeMatchingServers(s, servers)); - } - return serversToRemove; - } - /** * Verify the number of user regions is assigned to the primary * region server based on the plan is expected @@ -601,6 +469,7 @@ public class TestRegionPlacement { MetaScannerVisitor visitor = new MetaScannerVisitor() { public boolean processRow(Result result) throws IOException { try { + @SuppressWarnings("deprecation") HRegionInfo info = MetaScanner.getHRegionInfo(result); if(info.getTable().getNamespaceAsString() .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) { @@ -680,9 +549,12 @@ public class TestRegionPlacement { desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); admin.createTable(desc, splitKeys); + @SuppressWarnings("deprecation") HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName); + @SuppressWarnings("deprecation") Map regions = ht.getRegionLocations(); assertEquals("Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(), expectedRegions, regions.size()); + ht.close(); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement2.java new file mode 100644 index 00000000000..3f34bc49887 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement2.java @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import static org.junit.Assert.*; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer; +import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory; +import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan.Position; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({MasterTests.class, MediumTests.class}) +public class TestRegionPlacement2 { + final static Log LOG = LogFactory.getLog(TestRegionPlacement2.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final static int SLAVES = 7; + private final static int PRIMARY = Position.PRIMARY.ordinal(); + private final static int SECONDARY = Position.SECONDARY.ordinal(); + private final static int TERTIARY = Position.TERTIARY.ordinal(); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + // Enable the favored nodes based load balancer + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + FavoredNodeLoadBalancer.class, LoadBalancer.class); + conf.setBoolean("hbase.tests.use.shortcircuit.reads", false); + TEST_UTIL.startMiniCluster(SLAVES); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testFavoredNodesPresentForRoundRobinAssignment() throws HBaseIOException { + LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration()); + balancer.setMasterServices(TEST_UTIL.getMiniHBaseCluster().getMaster()); + List servers = new ArrayList(); + for (int i = 0; i < SLAVES; i++) { + ServerName server = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i).getServerName(); + servers.add(server); + } + List regions = new ArrayList(1); + HRegionInfo region = new HRegionInfo(TableName.valueOf("foobar")); + regions.add(region); + Map> assignmentMap = balancer.roundRobinAssignment(regions, + servers); + Set serverBefore = assignmentMap.keySet(); + List favoredNodesBefore = + ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region); + assertTrue(favoredNodesBefore.size() == 3); + // the primary RS should be the one that the balancer's assignment returns + assertTrue(ServerName.isSameHostnameAndPort(serverBefore.iterator().next(), + favoredNodesBefore.get(PRIMARY))); + // now remove the primary from the list of available servers + List removedServers = removeMatchingServers(serverBefore, servers); + // call roundRobinAssignment with the modified servers list + assignmentMap = balancer.roundRobinAssignment(regions, servers); + List favoredNodesAfter = + ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region); + assertTrue(favoredNodesAfter.size() == 3); + // We don't expect the favored nodes assignments to change in multiple calls + // to the roundRobinAssignment method in the balancer (relevant for AssignmentManager.assign + // failures) + assertTrue(favoredNodesAfter.containsAll(favoredNodesBefore)); + Set serverAfter = assignmentMap.keySet(); + // We expect the new RegionServer assignee to be one of the favored nodes + // chosen earlier. + assertTrue(ServerName.isSameHostnameAndPort(serverAfter.iterator().next(), + favoredNodesBefore.get(SECONDARY)) || + ServerName.isSameHostnameAndPort(serverAfter.iterator().next(), + favoredNodesBefore.get(TERTIARY))); + + // put back the primary in the list of available servers + servers.addAll(removedServers); + // now roundRobinAssignment with the modified servers list should return the primary + // as the regionserver assignee + assignmentMap = balancer.roundRobinAssignment(regions, servers); + Set serverWithPrimary = assignmentMap.keySet(); + assertTrue(serverBefore.containsAll(serverWithPrimary)); + + // Make all the favored nodes unavailable for assignment + removeMatchingServers(favoredNodesAfter, servers); + // call roundRobinAssignment with the modified servers list + assignmentMap = balancer.roundRobinAssignment(regions, servers); + List favoredNodesNow = + ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region); + assertTrue(favoredNodesNow.size() == 3); + assertTrue(!favoredNodesNow.contains(favoredNodesAfter.get(PRIMARY)) && + !favoredNodesNow.contains(favoredNodesAfter.get(SECONDARY)) && + !favoredNodesNow.contains(favoredNodesAfter.get(TERTIARY))); + } + + @Test + public void testFavoredNodesPresentForRandomAssignment() throws HBaseIOException { + LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration()); + balancer.setMasterServices(TEST_UTIL.getMiniHBaseCluster().getMaster()); + List servers = new ArrayList(); + for (int i = 0; i < SLAVES; i++) { + ServerName server = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i).getServerName(); + servers.add(server); + } + List regions = new ArrayList(1); + HRegionInfo region = new HRegionInfo(TableName.valueOf("foobar")); + regions.add(region); + ServerName serverBefore = balancer.randomAssignment(region, servers); + List favoredNodesBefore = + ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region); + assertTrue(favoredNodesBefore.size() == 3); + // the primary RS should be the one that the balancer's assignment returns + assertTrue(ServerName.isSameHostnameAndPort(serverBefore,favoredNodesBefore.get(PRIMARY))); + // now remove the primary from the list of servers + removeMatchingServers(serverBefore, servers); + // call randomAssignment with the modified servers list + ServerName serverAfter = balancer.randomAssignment(region, servers); + List favoredNodesAfter = + ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region); + assertTrue(favoredNodesAfter.size() == 3); + // We don't expect the favored nodes assignments to change in multiple calls + // to the randomAssignment method in the balancer (relevant for AssignmentManager.assign + // failures) + assertTrue(favoredNodesAfter.containsAll(favoredNodesBefore)); + // We expect the new RegionServer assignee to be one of the favored nodes + // chosen earlier. + assertTrue(ServerName.isSameHostnameAndPort(serverAfter, favoredNodesBefore.get(SECONDARY)) || + ServerName.isSameHostnameAndPort(serverAfter, favoredNodesBefore.get(TERTIARY))); + // Make all the favored nodes unavailable for assignment + removeMatchingServers(favoredNodesAfter, servers); + // call randomAssignment with the modified servers list + balancer.randomAssignment(region, servers); + List favoredNodesNow = + ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region); + assertTrue(favoredNodesNow.size() == 3); + assertTrue(!favoredNodesNow.contains(favoredNodesAfter.get(PRIMARY)) && + !favoredNodesNow.contains(favoredNodesAfter.get(SECONDARY)) && + !favoredNodesNow.contains(favoredNodesAfter.get(TERTIARY))); + } + + private List removeMatchingServers(Collection serversWithoutStartCode, + List servers) { + List serversToRemove = new ArrayList(); + for (ServerName s : serversWithoutStartCode) { + serversToRemove.addAll(removeMatchingServers(s, servers)); + } + return serversToRemove; + } + + private List removeMatchingServers(ServerName serverWithoutStartCode, + List servers) { + List serversToRemove = new ArrayList(); + for (ServerName s : servers) { + if (ServerName.isSameHostnameAndPort(s, serverWithoutStartCode)) { + serversToRemove.add(s); + } + } + servers.removeAll(serversToRemove); + return serversToRemove; + } +}