Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-18478 Allow users to remove RegionFinder from LoadBalancer calculations if no locality possible 

This provides significant cluster start time reduction for FileSystems which do not surface locality (S3).

Signed-off-by: Andrew Purtell <apurtell@apache.org>
This commit is contained in:
Zach York 2017-06-22 14:12:49 -07:00 committed by Andrew Purtell
parent bc790fe26a
commit 8301734594
1 changed files with 23 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
View File

@ -39,6 +39,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.HDFSBlocksDistribution;
@ -75,7 +76,8 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
static final Predicate<ServerLoad> IDLE_SERVER_PREDICATOR static final Predicate<ServerLoad> IDLE_SERVER_PREDICATOR
= load -> load.getNumberOfRegions() == 0; = load -> load.getNumberOfRegions() == 0;
protected final RegionLocationFinder regionFinder = new RegionLocationFinder(); protected RegionLocationFinder regionFinder;
protected boolean useRegionFinder;
private static class DefaultRackManager extends RackManager { private static class DefaultRackManager extends RackManager {
@Override @Override
@ -89,6 +91,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
*/ */
protected BaseLoadBalancer() { protected BaseLoadBalancer() {
metricsBalancer = new MetricsBalancer(); metricsBalancer = new MetricsBalancer();
createRegionFinder();
} }
/** /**
@ -97,6 +100,14 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
*/ */
protected BaseLoadBalancer(MetricsBalancer metricsBalancer) { protected BaseLoadBalancer(MetricsBalancer metricsBalancer) {
this.metricsBalancer = (metricsBalancer != null) ? metricsBalancer : new MetricsBalancer(); this.metricsBalancer = (metricsBalancer != null) ? metricsBalancer : new MetricsBalancer();
createRegionFinder();
}
private void createRegionFinder() {
useRegionFinder = config.getBoolean("hbase.master.balancer.uselocality", true);
if (useRegionFinder) {
regionFinder = new RegionLocationFinder();
}
} }
/** /**
@ -990,7 +1001,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
protected float slop; protected float slop;
// overallSlop to control simpleLoadBalancer's cluster level threshold // overallSlop to control simpleLoadBalancer's cluster level threshold
protected float overallSlop; protected float overallSlop;
protected Configuration config; protected Configuration config = HBaseConfiguration.create();
protected RackManager rackManager; protected RackManager rackManager;
private static final Random RANDOM = new Random(System.currentTimeMillis()); private static final Random RANDOM = new Random(System.currentTimeMillis());
private static final Log LOG = LogFactory.getLog(BaseLoadBalancer.class); private static final Log LOG = LogFactory.getLog(BaseLoadBalancer.class);
@ -1019,7 +1030,9 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
this.tablesOnMaster = true; this.tablesOnMaster = true;
} }
this.rackManager = new RackManager(getConf()); this.rackManager = new RackManager(getConf());
regionFinder.setConf(conf); if (useRegionFinder) {
regionFinder.setConf(conf);
}
// Print out base configs. Don't print overallSlop since it for simple balancer exclusively. // Print out base configs. Don't print overallSlop since it for simple balancer exclusively.
LOG.info("slop=" + this.slop + ", tablesOnMaster=" + this.tablesOnMaster + LOG.info("slop=" + this.slop + ", tablesOnMaster=" + this.tablesOnMaster +
", systemTablesOnMaster=" + this.onlySystemTablesOnMaster); ", systemTablesOnMaster=" + this.onlySystemTablesOnMaster);
@ -1117,7 +1130,9 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
@Override @Override
public synchronized void setClusterStatus(ClusterStatus st) { public synchronized void setClusterStatus(ClusterStatus st) {
this.clusterStatus = st; this.clusterStatus = st;
regionFinder.setClusterStatus(st); if (useRegionFinder) {
regionFinder.setClusterStatus(st);
}
} }
@Override @Override
@ -1129,7 +1144,9 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
public void setMasterServices(MasterServices masterServices) { public void setMasterServices(MasterServices masterServices) {
masterServerName = masterServices.getServerName(); masterServerName = masterServices.getServerName();
this.services = masterServices; this.services = masterServices;
this.regionFinder.setServices(masterServices); if (useRegionFinder) {
this.regionFinder.setServices(masterServices);
}
} }
public void setRackManager(RackManager rackManager) { public void setRackManager(RackManager rackManager) {
@ -1271,7 +1288,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
protected Cluster createCluster(List<ServerName> servers, protected Cluster createCluster(List<ServerName> servers,
Collection<HRegionInfo> regions, boolean forceRefresh) { Collection<HRegionInfo> regions, boolean forceRefresh) {
if (forceRefresh) { if (forceRefresh && useRegionFinder) {
regionFinder.refreshAndWait(regions); regionFinder.refreshAndWait(regions);
} }
// Get the snapshot of the current assignments for the regions in question, and then create // Get the snapshot of the current assignments for the regions in question, and then create