HBASE-25939 Move more tests code for StochasticLoadBalancer to hbase-balancer module (#3331)
Signed-off-by: Yulin Niu <niuyulin@apache.org>
This commit is contained in:
Duo Zhang
GitHub
parent
f119a865cf
commit
f2ff816532
|
|
@ -27,7 +27,10 @@ import org.apache.yetus.audience.InterfaceAudience;
|
|||
* The class that creates a load balancer from a conf.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class LoadBalancerFactory {
|
||||
public final class LoadBalancerFactory {
|
||||
|
||||
private LoadBalancerFactory() {
|
||||
}
|
||||
|
||||
/**
|
||||
* The default {@link LoadBalancer} class.
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.master.balancer;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.FileSystems;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.NoSuchFileException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
final class HeterogeneousCostRulesTestHelper {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(HeterogeneousCostRulesTestHelper.class);
|
||||
|
||||
static final String DEFAULT_RULES_FILE_NAME = "hbase-balancer.rules";
|
||||
|
||||
private HeterogeneousCostRulesTestHelper() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Create rule file with the given rules.
|
||||
* @param file Name of file to write rules into.
|
||||
* @return Full file name of the rules file which is <code>dir</code> + DEFAULT_RULES_FILE_NAME.
|
||||
*/
|
||||
static String createRulesFile(String file, final List<String> rules) throws IOException {
|
||||
cleanup(file);
|
||||
Path path = Files.createFile(FileSystems.getDefault().getPath(file));
|
||||
return Files.write(path, rules, StandardCharsets.UTF_8).toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Create rule file with empty rules.
|
||||
* @param file Name of file to write rules into.
|
||||
* @return Full file name of the rules file which is <code>dir</code> + DEFAULT_RULES_FILE_NAME.
|
||||
*/
|
||||
static String createRulesFile(String file) throws IOException {
|
||||
return createRulesFile(file, Collections.emptyList());
|
||||
}
|
||||
|
||||
static void cleanup(String file) throws IOException {
|
||||
try {
|
||||
Files.delete(FileSystems.getDefault().getPath(file));
|
||||
} catch (NoSuchFileException nsfe) {
|
||||
LOG.warn("FileNotFoundException for {}", file, nsfe);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -87,7 +87,8 @@ public class LoadBalancerPerformanceEvaluation extends AbstractHBaseTool {
|
|||
|
||||
// Non-default configurations.
|
||||
private void setupConf() {
|
||||
conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, loadBalancerClazz, LoadBalancer.class);
|
||||
conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, loadBalancerClazz,
|
||||
LoadBalancer.class);
|
||||
loadBalancer = LoadBalancerFactory.getLoadBalancer(conf);
|
||||
}
|
||||
|
||||
|
|
@ -1,27 +1,32 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
|
||||
* agreements. See the NOTICE file distributed with this work for additional information regarding
|
||||
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance with the License. You may obtain a
|
||||
* copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
* or implied. See the License for the specific language governing permissions and limitations under
|
||||
* the License.
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.master.balancer;
|
||||
|
||||
import static junit.framework.TestCase.assertNotNull;
|
||||
import static junit.framework.TestCase.assertTrue;
|
||||
import static org.apache.hadoop.hbase.master.balancer.HeterogeneousCostRulesTestHelper.DEFAULT_RULES_FILE_NAME;
|
||||
import static org.apache.hadoop.hbase.master.balancer.HeterogeneousCostRulesTestHelper.createRulesFile;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Queue;
|
||||
|
|
@ -30,7 +35,7 @@ import java.util.TreeMap;
|
|||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.client.RegionInfo;
|
||||
|
|
@ -48,14 +53,15 @@ import org.slf4j.LoggerFactory;
|
|||
|
||||
@Category({ MasterTests.class, MediumTests.class })
|
||||
public class TestStochasticLoadBalancerHeterogeneousCost extends StochasticBalancerTestBase {
|
||||
|
||||
@ClassRule
|
||||
public static final HBaseClassTestRule CLASS_RULE =
|
||||
HBaseClassTestRule.forClass(TestStochasticLoadBalancerHeterogeneousCost.class);
|
||||
HBaseClassTestRule.forClass(TestStochasticLoadBalancerHeterogeneousCost.class);
|
||||
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(TestStochasticLoadBalancerHeterogeneousCost.class);
|
||||
LoggerFactory.getLogger(TestStochasticLoadBalancerHeterogeneousCost.class);
|
||||
private static final double ALLOWED_WINDOW = 1.20;
|
||||
private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
|
||||
private static final HBaseCommonTestingUtility HTU = new HBaseCommonTestingUtility();
|
||||
private static String RULES_FILE;
|
||||
|
||||
@BeforeClass
|
||||
|
|
@ -69,10 +75,8 @@ public class TestStochasticLoadBalancerHeterogeneousCost extends StochasticBalan
|
|||
HeterogeneousRegionCountCostFunction.class.getName());
|
||||
// Need to ensure test dir has been created.
|
||||
assertTrue(FileSystem.get(HTU.getConfiguration()).mkdirs(HTU.getDataTestDir()));
|
||||
RULES_FILE = HTU.getDataTestDir(
|
||||
TestStochasticLoadBalancerHeterogeneousCostRules.DEFAULT_RULES_FILE_NAME).toString();
|
||||
conf.set(
|
||||
HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
|
||||
RULES_FILE = HTU.getDataTestDir(DEFAULT_RULES_FILE_NAME).toString();
|
||||
conf.set(HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
|
||||
RULES_FILE);
|
||||
loadBalancer = new StochasticLoadBalancer();
|
||||
loadBalancer.setClusterInfoProvider(new DummyClusterInfoProvider(conf));
|
||||
|
|
@ -146,28 +150,29 @@ public class TestStochasticLoadBalancerHeterogeneousCost extends StochasticBalan
|
|||
final int numRegions = 120;
|
||||
final int numRegionsPerServer = 60;
|
||||
|
||||
TestStochasticLoadBalancerHeterogeneousCostRules.createRulesFile(RULES_FILE);
|
||||
createRulesFile(RULES_FILE);
|
||||
final Map<ServerName, List<RegionInfo>> serverMap =
|
||||
this.createServerMap(numNodes, numRegions, numRegionsPerServer, 1, 1);
|
||||
this.createServerMap(numNodes, numRegions, numRegionsPerServer, 1, 1);
|
||||
final List<RegionPlan> plans =
|
||||
loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap);
|
||||
loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap);
|
||||
// As we disabled all the other cost functions, balancing only according to
|
||||
// the heterogeneous cost function should return nothing.
|
||||
assertNull(plans);
|
||||
}
|
||||
|
||||
private void testHeterogeneousWithCluster(final int numNodes, final int numRegions,
|
||||
final int numRegionsPerServer, final List<String> rules) throws IOException {
|
||||
final int numRegionsPerServer, final List<String> rules) throws IOException {
|
||||
|
||||
TestStochasticLoadBalancerHeterogeneousCostRules.createRulesFile(RULES_FILE, rules);
|
||||
createRulesFile(RULES_FILE, rules);
|
||||
final Map<ServerName, List<RegionInfo>> serverMap =
|
||||
this.createServerMap(numNodes, numRegions, numRegionsPerServer, 1, 1);
|
||||
this.createServerMap(numNodes, numRegions, numRegionsPerServer, 1, 1);
|
||||
this.testWithCluster(serverMap, null, true, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void testWithCluster(final Map<ServerName, List<RegionInfo>> serverMap,
|
||||
final RackManager rackManager, final boolean assertFullyBalanced,
|
||||
final boolean assertFullyBalancedForReplicas) {
|
||||
final RackManager rackManager, final boolean assertFullyBalanced,
|
||||
final boolean assertFullyBalancedForReplicas) {
|
||||
final List<ServerAndLoad> list = this.convertToList(serverMap);
|
||||
LOG.info("Mock Cluster : " + this.printMock(list) + " " + this.printStats(list));
|
||||
|
||||
|
|
@ -175,7 +180,7 @@ public class TestStochasticLoadBalancerHeterogeneousCost extends StochasticBalan
|
|||
|
||||
// Run the balancer.
|
||||
final List<RegionPlan> plans =
|
||||
loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap);
|
||||
loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap);
|
||||
assertNotNull(plans);
|
||||
|
||||
// Check to see that this actually got to a stable place.
|
||||
|
|
@ -188,16 +193,15 @@ public class TestStochasticLoadBalancerHeterogeneousCost extends StochasticBalan
|
|||
|
||||
if (assertFullyBalanced) {
|
||||
final List<RegionPlan> secondPlans =
|
||||
loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap);
|
||||
loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap);
|
||||
assertNull(secondPlans);
|
||||
|
||||
// create external cost function to retrieve limit
|
||||
// for each RS
|
||||
final HeterogeneousRegionCountCostFunction cf =
|
||||
new HeterogeneousRegionCountCostFunction(conf);
|
||||
new HeterogeneousRegionCountCostFunction(conf);
|
||||
assertNotNull(cf);
|
||||
BalancerClusterState cluster =
|
||||
new BalancerClusterState(serverMap, null, null, null);
|
||||
BalancerClusterState cluster = new BalancerClusterState(serverMap, null, null, null);
|
||||
cf.prepare(cluster);
|
||||
|
||||
// checking that we all hosts have a number of regions below their limit
|
||||
|
|
@ -212,10 +216,9 @@ public class TestStochasticLoadBalancerHeterogeneousCost extends StochasticBalan
|
|||
|
||||
// as the balancer is stochastic, we cannot check exactly the result of the balancing,
|
||||
// hence the allowedWindow parameter
|
||||
assertTrue("Host " + sn.getHostname() + " should be below "
|
||||
+ cf.overallUsage * ALLOWED_WINDOW * 100 + "%; " + cf.overallUsage +
|
||||
", " + usage + ", " + numberRegions + ", " + limit,
|
||||
usage <= cf.overallUsage * ALLOWED_WINDOW);
|
||||
assertTrue("Host " + sn.getHostname() + " should be below " +
|
||||
cf.overallUsage * ALLOWED_WINDOW * 100 + "%; " + cf.overallUsage + ", " + usage + ", " +
|
||||
numberRegions + ", " + limit, usage <= cf.overallUsage * ALLOWED_WINDOW);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -227,7 +230,7 @@ public class TestStochasticLoadBalancerHeterogeneousCost extends StochasticBalan
|
|||
|
||||
@Override
|
||||
protected Map<ServerName, List<RegionInfo>> createServerMap(int numNodes, int numRegions,
|
||||
int numRegionsPerServer, int replication, int numTables) {
|
||||
int numRegionsPerServer, int replication, int numTables) {
|
||||
// construct a cluster of numNodes, having a total of numRegions. Each RS will hold
|
||||
// numRegionsPerServer many regions except for the last one, which will host all the
|
||||
// remaining regions
|
||||
|
|
@ -254,7 +257,7 @@ public class TestStochasticLoadBalancerHeterogeneousCost extends StochasticBalan
|
|||
|
||||
@Override
|
||||
protected TreeMap<ServerName, List<RegionInfo>> mockClusterServers(int[] mockCluster,
|
||||
int numTables) {
|
||||
int numTables) {
|
||||
int numServers = mockCluster.length;
|
||||
TreeMap<ServerName, List<RegionInfo>> servers = new TreeMap<>();
|
||||
for (int i = 0; i < numServers; i++) {
|
||||
|
|
@ -266,7 +269,7 @@ public class TestStochasticLoadBalancerHeterogeneousCost extends StochasticBalan
|
|||
return servers;
|
||||
}
|
||||
|
||||
private Queue<ServerName> serverQueue = new LinkedList<>();
|
||||
private Queue<ServerName> serverQueue = new ArrayDeque<>();
|
||||
|
||||
private ServerAndLoad createServer(final String host) {
|
||||
if (!this.serverQueue.isEmpty()) {
|
||||
|
|
@ -280,12 +283,11 @@ public class TestStochasticLoadBalancerHeterogeneousCost extends StochasticBalan
|
|||
return new ServerAndLoad(sn, 0);
|
||||
}
|
||||
|
||||
static class FairRandomCandidateGenerator extends
|
||||
RandomCandidateGenerator {
|
||||
static class FairRandomCandidateGenerator extends RandomCandidateGenerator {
|
||||
|
||||
@Override
|
||||
public BalanceAction pickRandomRegions(BalancerClusterState cluster,
|
||||
int thisServer, int otherServer) {
|
||||
public BalanceAction pickRandomRegions(BalancerClusterState cluster, int thisServer,
|
||||
int otherServer) {
|
||||
if (thisServer < 0 || otherServer < 0) {
|
||||
return BalanceAction.NULL_ACTION;
|
||||
}
|
||||
|
|
@ -301,4 +303,4 @@ public class TestStochasticLoadBalancerHeterogeneousCost extends StochasticBalan
|
|||
return super.generate(cluster);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,36 +1,34 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license
|
||||
* agreements. See the NOTICE file distributed with this work for additional information regarding
|
||||
* copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance with the License. You may obtain a
|
||||
* copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
* or implied. See the License for the specific language governing permissions and limitations under
|
||||
* the License.
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.master.balancer;
|
||||
|
||||
import static org.apache.hadoop.hbase.master.balancer.HeterogeneousCostRulesTestHelper.DEFAULT_RULES_FILE_NAME;
|
||||
import static org.apache.hadoop.hbase.master.balancer.HeterogeneousCostRulesTestHelper.cleanup;
|
||||
import static org.apache.hadoop.hbase.master.balancer.HeterogeneousCostRulesTestHelper.createRulesFile;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.file.FileSystems;
|
||||
import java.nio.file.NoSuchFileException;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
|
||||
import org.apache.hadoop.hbase.testclassification.MasterTests;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.ClassRule;
|
||||
|
|
@ -41,15 +39,15 @@ import org.junit.rules.TestName;
|
|||
|
||||
@Category({ MasterTests.class, MediumTests.class })
|
||||
public class TestStochasticLoadBalancerHeterogeneousCostRules extends StochasticBalancerTestBase {
|
||||
|
||||
@ClassRule
|
||||
public static final HBaseClassTestRule CLASS_RULE =
|
||||
HBaseClassTestRule.forClass(TestStochasticLoadBalancerHeterogeneousCostRules.class);
|
||||
@Rule
|
||||
public TestName name = new TestName();
|
||||
|
||||
static final String DEFAULT_RULES_FILE_NAME = "hbase-balancer.rules";
|
||||
private HeterogeneousRegionCountCostFunction costFunction;
|
||||
private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
|
||||
private static final HBaseCommonTestingUtility HTU = new HBaseCommonTestingUtility();
|
||||
|
||||
/**
|
||||
* Make a file for rules that is inside a temporary test dir named for the method so it doesn't
|
||||
|
|
@ -60,47 +58,22 @@ public class TestStochasticLoadBalancerHeterogeneousCostRules extends Stochastic
|
|||
@BeforeClass
|
||||
public static void beforeClass() throws IOException {
|
||||
// Ensure test dir is created
|
||||
HTU.getTestFileSystem().mkdirs(HTU.getDataTestDir());
|
||||
HTU.getDataTestDir().getFileSystem(HTU.getConfiguration()).mkdirs(HTU.getDataTestDir());
|
||||
}
|
||||
|
||||
@Before
|
||||
public void before() throws IOException {
|
||||
// New rules file name per test.
|
||||
this.rulesFilename = HTU.getDataTestDir(
|
||||
this.name.getMethodName() + "." + DEFAULT_RULES_FILE_NAME).toString();
|
||||
this.rulesFilename = HTU
|
||||
.getDataTestDir(
|
||||
this.name.getMethodName() + "." + DEFAULT_RULES_FILE_NAME)
|
||||
.toString();
|
||||
// Set the created rules filename into the configuration.
|
||||
HTU.getConfiguration().set(
|
||||
HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
|
||||
this.rulesFilename);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param file Name of file to write rules into.
|
||||
* @return Full file name of the rules file which is <code>dir</code> + DEFAULT_RULES_FILE_NAME.
|
||||
*/
|
||||
static String createRulesFile(String file, final List<String> lines) throws IOException {
|
||||
cleanup(file);
|
||||
java.nio.file.Path path =
|
||||
java.nio.file.Files.createFile(FileSystems.getDefault().getPath(file));
|
||||
return java.nio.file.Files.write(path, lines, Charset.forName("UTF-8")).toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param file Name of file to write rules into.
|
||||
* @return Full file name of the rules file which is <code>dir</code> + DEFAULT_RULES_FILE_NAME.
|
||||
*/
|
||||
static String createRulesFile(String file) throws IOException {
|
||||
return createRulesFile(file, Collections.emptyList());
|
||||
}
|
||||
|
||||
private static void cleanup(String file) throws IOException {
|
||||
try {
|
||||
java.nio.file.Files.delete(FileSystems.getDefault().getPath(file));
|
||||
} catch (NoSuchFileException nsfe) {
|
||||
System.out.println("FileNotFoundException for " + file);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoRules() throws IOException {
|
||||
// Override what is in the configuration with the name of a non-existent file!
|
||||
|
|
@ -109,7 +82,7 @@ public class TestStochasticLoadBalancerHeterogeneousCostRules extends Stochastic
|
|||
"non-existent-file!");
|
||||
this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
|
||||
this.costFunction.loadRules();
|
||||
Assert.assertEquals(0, this.costFunction.getNumberOfRulesLoaded());
|
||||
assertEquals(0, this.costFunction.getNumberOfRulesLoaded());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -119,18 +92,18 @@ public class TestStochasticLoadBalancerHeterogeneousCostRules extends Stochastic
|
|||
// in the configuration.
|
||||
this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
|
||||
this.costFunction.loadRules();
|
||||
Assert.assertEquals(0, this.costFunction.getNumberOfRulesLoaded());
|
||||
assertEquals(0, this.costFunction.getNumberOfRulesLoaded());
|
||||
|
||||
createRulesFile(this.rulesFilename, Collections.singletonList("bad rules format"));
|
||||
this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
|
||||
this.costFunction.loadRules();
|
||||
Assert.assertEquals(0, this.costFunction.getNumberOfRulesLoaded());
|
||||
assertEquals(0, this.costFunction.getNumberOfRulesLoaded());
|
||||
|
||||
createRulesFile(this.rulesFilename, Arrays.asList("srv[1-2] 10",
|
||||
"bad_rules format", "a"));
|
||||
this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
|
||||
this.costFunction.loadRules();
|
||||
Assert.assertEquals(1, this.costFunction.getNumberOfRulesLoaded());
|
||||
assertEquals(1, this.costFunction.getNumberOfRulesLoaded());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -144,7 +117,7 @@ public class TestStochasticLoadBalancerHeterogeneousCostRules extends Stochastic
|
|||
createRulesFile(this.rulesFilename, Arrays.asList("^server1$ 10", "^server2 21"));
|
||||
this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
|
||||
this.costFunction.loadRules();
|
||||
Assert.assertEquals(2, this.costFunction.getNumberOfRulesLoaded());
|
||||
assertEquals(2, this.costFunction.getNumberOfRulesLoaded());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -158,7 +131,7 @@ public class TestStochasticLoadBalancerHeterogeneousCostRules extends Stochastic
|
|||
createRulesFile(this.rulesFilename, Collections.singletonList("server[ 1"));
|
||||
this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
|
||||
this.costFunction.loadRules();
|
||||
Assert.assertEquals(0, this.costFunction.getNumberOfRulesLoaded());
|
||||
assertEquals(0, this.costFunction.getNumberOfRulesLoaded());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -169,38 +142,11 @@ public class TestStochasticLoadBalancerHeterogeneousCostRules extends Stochastic
|
|||
createRulesFile(this.rulesFilename, Arrays.asList("^server1$ 10", "^server2 21"));
|
||||
this.costFunction = new HeterogeneousRegionCountCostFunction(HTU.getConfiguration());
|
||||
this.costFunction.loadRules();
|
||||
Assert.assertEquals(2, this.costFunction.getNumberOfRulesLoaded());
|
||||
assertEquals(2, this.costFunction.getNumberOfRulesLoaded());
|
||||
|
||||
// loading malformed configuration does not overload current
|
||||
cleanup(this.rulesFilename);
|
||||
this.costFunction.loadRules();
|
||||
Assert.assertEquals(2, this.costFunction.getNumberOfRulesLoaded());
|
||||
assertEquals(2, this.costFunction.getNumberOfRulesLoaded());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLoadingFomHDFS() throws Exception {
|
||||
HTU.startMiniDFSCluster(3);
|
||||
try {
|
||||
MiniDFSCluster cluster = HTU.getDFSCluster();
|
||||
DistributedFileSystem fs = cluster.getFileSystem();
|
||||
// Writing file
|
||||
Path path = new Path(fs.getHomeDirectory(), DEFAULT_RULES_FILE_NAME);
|
||||
FSDataOutputStream stream = fs.create(path);
|
||||
stream.write("server1 10".getBytes());
|
||||
stream.flush();
|
||||
stream.close();
|
||||
|
||||
Configuration configuration = HTU.getConfiguration();
|
||||
|
||||
// start costFunction
|
||||
configuration.set(
|
||||
HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
|
||||
path.toString());
|
||||
this.costFunction = new HeterogeneousRegionCountCostFunction(configuration);
|
||||
this.costFunction.loadRules();
|
||||
Assert.assertEquals(1, this.costFunction.getNumberOfRulesLoaded());
|
||||
} finally {
|
||||
HTU.shutdownMiniCluster();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.master.balancer;
|
||||
|
||||
import static org.apache.hadoop.hbase.master.balancer.HeterogeneousCostRulesTestHelper.DEFAULT_RULES_FILE_NAME;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import java.io.IOException;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.testclassification.MasterTests;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
|
||||
@Category({ MasterTests.class, MediumTests.class })
|
||||
public class TestStochasticLoadBalancerHeterogeneousCostRulesLoadFromHDFS
|
||||
extends StochasticBalancerTestBase {
|
||||
|
||||
@ClassRule
|
||||
public static final HBaseClassTestRule CLASS_RULE =
|
||||
HBaseClassTestRule.forClass(TestStochasticLoadBalancerHeterogeneousCostRulesLoadFromHDFS.class);
|
||||
|
||||
private HeterogeneousRegionCountCostFunction costFunction;
|
||||
private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
HTU.startMiniCluster(1);
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws IOException {
|
||||
HTU.shutdownMiniCluster();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLoadingFomHDFS() throws Exception {
|
||||
MiniDFSCluster cluster = HTU.getDFSCluster();
|
||||
DistributedFileSystem fs = cluster.getFileSystem();
|
||||
// Writing file
|
||||
Path path = new Path(fs.getHomeDirectory(), DEFAULT_RULES_FILE_NAME);
|
||||
FSDataOutputStream stream = fs.create(path);
|
||||
stream.write("server1 10".getBytes());
|
||||
stream.flush();
|
||||
stream.close();
|
||||
|
||||
Configuration configuration = HTU.getConfiguration();
|
||||
|
||||
// start costFunction
|
||||
configuration.set(
|
||||
HeterogeneousRegionCountCostFunction.HBASE_MASTER_BALANCER_HETEROGENEOUS_RULES_FILE,
|
||||
path.toString());
|
||||
this.costFunction = new HeterogeneousRegionCountCostFunction(configuration);
|
||||
this.costFunction.loadRules();
|
||||
assertEquals(1, this.costFunction.getNumberOfRulesLoaded());
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue