HBASE-18478 Allow users to remove RegionFinder from LoadBalancer calculations if no locality possible
This provides significant cluster start time reduction for FileSystems which do not surface locality (S3). Signed-off-by: Andrew Purtell <apurtell@apache.org>
This commit is contained in:
parent
bc790fe26a
commit
8301734594
|
@ -39,6 +39,7 @@ import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.hbase.ClusterStatus;
|
import org.apache.hadoop.hbase.ClusterStatus;
|
||||||
|
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||||
import org.apache.hadoop.hbase.HBaseIOException;
|
import org.apache.hadoop.hbase.HBaseIOException;
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
|
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
|
||||||
|
@ -75,7 +76,8 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
||||||
static final Predicate<ServerLoad> IDLE_SERVER_PREDICATOR
|
static final Predicate<ServerLoad> IDLE_SERVER_PREDICATOR
|
||||||
= load -> load.getNumberOfRegions() == 0;
|
= load -> load.getNumberOfRegions() == 0;
|
||||||
|
|
||||||
protected final RegionLocationFinder regionFinder = new RegionLocationFinder();
|
protected RegionLocationFinder regionFinder;
|
||||||
|
protected boolean useRegionFinder;
|
||||||
|
|
||||||
private static class DefaultRackManager extends RackManager {
|
private static class DefaultRackManager extends RackManager {
|
||||||
@Override
|
@Override
|
||||||
|
@ -89,6 +91,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
||||||
*/
|
*/
|
||||||
protected BaseLoadBalancer() {
|
protected BaseLoadBalancer() {
|
||||||
metricsBalancer = new MetricsBalancer();
|
metricsBalancer = new MetricsBalancer();
|
||||||
|
createRegionFinder();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -97,6 +100,14 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
||||||
*/
|
*/
|
||||||
protected BaseLoadBalancer(MetricsBalancer metricsBalancer) {
|
protected BaseLoadBalancer(MetricsBalancer metricsBalancer) {
|
||||||
this.metricsBalancer = (metricsBalancer != null) ? metricsBalancer : new MetricsBalancer();
|
this.metricsBalancer = (metricsBalancer != null) ? metricsBalancer : new MetricsBalancer();
|
||||||
|
createRegionFinder();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void createRegionFinder() {
|
||||||
|
useRegionFinder = config.getBoolean("hbase.master.balancer.uselocality", true);
|
||||||
|
if (useRegionFinder) {
|
||||||
|
regionFinder = new RegionLocationFinder();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -990,7 +1001,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
||||||
protected float slop;
|
protected float slop;
|
||||||
// overallSlop to control simpleLoadBalancer's cluster level threshold
|
// overallSlop to control simpleLoadBalancer's cluster level threshold
|
||||||
protected float overallSlop;
|
protected float overallSlop;
|
||||||
protected Configuration config;
|
protected Configuration config = HBaseConfiguration.create();
|
||||||
protected RackManager rackManager;
|
protected RackManager rackManager;
|
||||||
private static final Random RANDOM = new Random(System.currentTimeMillis());
|
private static final Random RANDOM = new Random(System.currentTimeMillis());
|
||||||
private static final Log LOG = LogFactory.getLog(BaseLoadBalancer.class);
|
private static final Log LOG = LogFactory.getLog(BaseLoadBalancer.class);
|
||||||
|
@ -1019,7 +1030,9 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
||||||
this.tablesOnMaster = true;
|
this.tablesOnMaster = true;
|
||||||
}
|
}
|
||||||
this.rackManager = new RackManager(getConf());
|
this.rackManager = new RackManager(getConf());
|
||||||
|
if (useRegionFinder) {
|
||||||
regionFinder.setConf(conf);
|
regionFinder.setConf(conf);
|
||||||
|
}
|
||||||
// Print out base configs. Don't print overallSlop since it for simple balancer exclusively.
|
// Print out base configs. Don't print overallSlop since it for simple balancer exclusively.
|
||||||
LOG.info("slop=" + this.slop + ", tablesOnMaster=" + this.tablesOnMaster +
|
LOG.info("slop=" + this.slop + ", tablesOnMaster=" + this.tablesOnMaster +
|
||||||
", systemTablesOnMaster=" + this.onlySystemTablesOnMaster);
|
", systemTablesOnMaster=" + this.onlySystemTablesOnMaster);
|
||||||
|
@ -1117,8 +1130,10 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
||||||
@Override
|
@Override
|
||||||
public synchronized void setClusterStatus(ClusterStatus st) {
|
public synchronized void setClusterStatus(ClusterStatus st) {
|
||||||
this.clusterStatus = st;
|
this.clusterStatus = st;
|
||||||
|
if (useRegionFinder) {
|
||||||
regionFinder.setClusterStatus(st);
|
regionFinder.setClusterStatus(st);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setClusterLoad(Map<TableName, Map<ServerName, List<HRegionInfo>>> clusterLoad){
|
public void setClusterLoad(Map<TableName, Map<ServerName, List<HRegionInfo>>> clusterLoad){
|
||||||
|
@ -1129,8 +1144,10 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
||||||
public void setMasterServices(MasterServices masterServices) {
|
public void setMasterServices(MasterServices masterServices) {
|
||||||
masterServerName = masterServices.getServerName();
|
masterServerName = masterServices.getServerName();
|
||||||
this.services = masterServices;
|
this.services = masterServices;
|
||||||
|
if (useRegionFinder) {
|
||||||
this.regionFinder.setServices(masterServices);
|
this.regionFinder.setServices(masterServices);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public void setRackManager(RackManager rackManager) {
|
public void setRackManager(RackManager rackManager) {
|
||||||
this.rackManager = rackManager;
|
this.rackManager = rackManager;
|
||||||
|
@ -1271,7 +1288,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
||||||
|
|
||||||
protected Cluster createCluster(List<ServerName> servers,
|
protected Cluster createCluster(List<ServerName> servers,
|
||||||
Collection<HRegionInfo> regions, boolean forceRefresh) {
|
Collection<HRegionInfo> regions, boolean forceRefresh) {
|
||||||
if (forceRefresh) {
|
if (forceRefresh && useRegionFinder) {
|
||||||
regionFinder.refreshAndWait(regions);
|
regionFinder.refreshAndWait(regions);
|
||||||
}
|
}
|
||||||
// Get the snapshot of the current assignments for the regions in question, and then create
|
// Get the snapshot of the current assignments for the regions in question, and then create
|
||||||
|
|
Loading…
Reference in New Issue