Apache/hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-12559 Provide LoadBalancer with online configuration capability

Browse Source
This commit is contained in:
tedyu 2014-12-01 14:11:01 -08:00
parent 2c280e6253
commit a30cca91e0
7 changed files with 216 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
View File

@ -660,6 +660,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
status.markComplete("Initialization successful");
LOG.info("Master has completed initialization");
configurationManager.registerObserver(this.balancer);
initialized = true;
// clear the dead servers with same host name and port of online server because we are not
// removing dead server with same hostname and port of rs which is trying to check in before

10
hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
View File

@ -22,7 +22,9 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.conf.ConfigurationObserver;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HRegionInfo;
@ -46,7 +48,7 @@ import org.apache.hadoop.hbase.Stoppable;
* <p>This classes produces plans for the {@link AssignmentManager} to execute.
*/
@InterfaceAudience.Private
public interface LoadBalancer extends Configurable, Stoppable {
public interface LoadBalancer extends Configurable, Stoppable, ConfigurationObserver {
/**
* Set the current cluster status. This allows a LoadBalancer to map host name to a server
@ -130,4 +132,10 @@ public interface LoadBalancer extends Configurable, Stoppable {
* @param regionInfo
*/
void regionOffline(HRegionInfo regionInfo);
/*
* Notification that config has changed
* @param conf
*/
void onConfigurationChange(Configuration conf);
}

5
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
View File

@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.conf.ConfigurationObserver;
import org.apache.hadoop.hbase.master.LoadBalancer;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RackManager;
@ -1388,4 +1389,8 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
return new HashMap<ServerName, List<HRegionInfo>>();
}
}
@Override
public void onConfigurationChange(Configuration conf) {
}
}

26
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
View File

@ -125,8 +125,14 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
private RegionReplicaRackCostFunction regionReplicaRackCostFunction;
@Override
public void setConf(Configuration conf) {
public void onConfigurationChange(Configuration conf) {
setConf(conf);
}
@Override
public synchronized void setConf(Configuration conf) {
super.setConf(conf);
LOG.info("loading config");
maxSteps = conf.getInt(MAX_STEPS_KEY, maxSteps);
@ -135,15 +141,19 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, numRegionLoadsToRemember);
localityCandidateGenerator = new LocalityBasedCandidateGenerator(services);
if (localityCandidateGenerator == null) {
localityCandidateGenerator = new LocalityBasedCandidateGenerator(services);
}
localityCost = new LocalityCostFunction(conf, services);
candidateGenerators = new CandidateGenerator[] {
new RandomCandidateGenerator(),
new LoadCandidateGenerator(),
localityCandidateGenerator,
new RegionReplicaRackCandidateGenerator(),
};
if (candidateGenerators == null) {
candidateGenerators = new CandidateGenerator[] {
new RandomCandidateGenerator(),
new LoadCandidateGenerator(),
localityCandidateGenerator,
new RegionReplicaRackCandidateGenerator(),
};
}
regionLoadFunctions = new CostFromRegionLoadFunction[] {
new ReadRequestCostFunction(conf),

2
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
View File

@ -442,7 +442,7 @@ public class HRegionServer extends HasThread implements
* Configuration manager is used to register/deregister and notify the configuration observers
* when the regionserver is notified that there was a change in the on disk configs.
*/
private final ConfigurationManager configurationManager;
protected final ConfigurationManager configurationManager;
/**
* Starts a HRegionServer at the default location.

28
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
View File

@ -18,10 +18,17 @@
*/
package org.apache.hadoop.hbase.client;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@ -46,4 +53,25 @@ public class TestUpdateConfiguration {
ServerName server = TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName();
admin.updateConfiguration(server);
}
@Test
public void testMasterOnlineConfigChange() throws IOException {
LOG.debug("Starting the test");
Path cnfPath = FileSystems.getDefault().getPath("target/test-classes/hbase-site.xml");
Path cnf2Path = FileSystems.getDefault().getPath("target/test-classes/hbase-site2.xml");
Path cnf3Path = FileSystems.getDefault().getPath("target/test-classes/hbase-site3.xml");
// make a backup of hbase-site.xml
Files.copy(cnfPath, cnf3Path, StandardCopyOption.REPLACE_EXISTING);
// update hbase-site.xml by overwriting it
Files.copy(cnf2Path, cnfPath, StandardCopyOption.REPLACE_EXISTING);
Admin admin = TEST_UTIL.getHBaseAdmin();
ServerName server = TEST_UTIL.getHBaseCluster().getMaster().getServerName();
admin.updateConfiguration(server);
Configuration conf = TEST_UTIL.getMiniHBaseCluster().getMaster().getConfiguration();
int custom = conf.getInt("hbase.custom.config", 0);
assertEquals(custom, 1000);
// restore hbase-site.xml
Files.copy(cnf3Path, cnfPath, StandardCopyOption.REPLACE_EXISTING);
}
}

154
hbase-server/src/test/resources/hbase-site2.xml Normal file
View File

@ -0,0 +1,154 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-->
<configuration>
<property>
<name>hbase.custom.config</name>
<value>1000</value>
</property>
<property>
<name>hbase.regionserver.msginterval</name>
<value>1000</value>
<description>Interval between messages from the RegionServer to HMaster
in milliseconds. Default is 15. Set this value low if you want unit
tests to be responsive.
</description>
</property>
<property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
</property>
<property>
<name>hbase.server.thread.wakefrequency</name>
<value>1000</value>
<description>Time to sleep in between searches for work (in milliseconds).
Used as sleep interval by service threads such as hbase:meta scanner and log roller.
</description>
</property>
<property>
<name>hbase.master.event.waiting.time</name>
<value>50</value>
<description>Time to sleep between checks to see if a table event took place.
</description>
</property>
<property>
<name>hbase.regionserver.handler.count</name>
<value>5</value>
</property>
<property>
<name>hbase.master.info.port</name>
<value>-1</value>
<description>The port for the hbase master web UI
Set to -1 if you do not want the info server to run.
</description>
</property>
<property>
<name>hbase.master.port</name>
<value>0</value>
<description>Always have masters and regionservers come up on port '0' so we don't clash over
default ports.
</description>
</property>
<property>
<name>hbase.regionserver.port</name>
<value>0</value>
<description>Always have masters and regionservers come up on port '0' so we don't clash over
default ports.
</description>
</property>
<property>
<name>hbase.ipc.client.fallback-to-simple-auth-allowed</name>
<value>true</value>
</property>
<property>
<name>hbase.regionserver.info.port</name>
<value>-1</value>
<description>The port for the hbase regionserver web UI
Set to -1 if you do not want the info server to run.
</description>
</property>
<property>
<name>hbase.regionserver.info.port.auto</name>
<value>true</value>
<description>Info server auto port bind. Enables automatic port
search if hbase.regionserver.info.port is already in use.
Enabled for testing to run multiple tests on one machine.
</description>
</property>
<property>
<name>hbase.master.lease.thread.wakefrequency</name>
<value>3000</value>
<description>The interval between checks for expired region server leases.
This value has been reduced due to the other reduced values above so that
the master will notice a dead region server sooner. The default is 15 seconds.
</description>
</property>
<property>
<name>hbase.regionserver.safemode</name>
<value>false</value>
<description>
Turn on/off safe mode in region server. Always on for production, always off
for tests.
</description>
</property>
<property>
<name>hbase.hregion.max.filesize</name>
<value>67108864</value>
<description>
Maximum desired file size for an HRegion. If filesize exceeds
value + (value / 2), the HRegion is split in two. Default: 256M.
Keep the maximum filesize small so we split more often in tests.
</description>
</property>
<property>
<name>hadoop.log.dir</name>
<value>${user.dir}/../logs</value>
</property>
<property>
<name>hbase.zookeeper.property.clientPort</name>
<value>21818</value>
<description>Property from ZooKeeper's config zoo.cfg.
The port at which the clients will connect.
</description>
</property>
<property>
<name>hbase.defaults.for.version.skip</name>
<value>true</value>
<description>
Set to true to skip the 'hbase.defaults.for.version'.
Setting this to true can be useful in contexts other than
the other side of a maven generation; i.e. running in an
ide. You'll want to set this boolean to true to avoid
seeing the RuntimException complaint: "hbase-default.xml file
seems to be for and old version of HBase (@@@VERSION@@@), this
version is X.X.X-SNAPSHOT"
</description>
</property>
<property>
<name>hbase.table.sanity.checks</name>
<value>false</value>
<description>Skip sanity checks in tests
</description>
</property>
</configuration>