HBASE-25290 Remove table on master related code in balancer implementation (#3162)
Signed-off-by: Yulin Niu <niuyulin@apache.org>
This commit is contained in:
Duo Zhang
GitHub
parent
33e886c6cc
commit
781da1899a
@ -26,13 +26,9 @@ import java.io.FileReader;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
|
||||
import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZNodePaths;
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
import org.apache.zookeeper.KeeperException;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
@ -145,21 +141,6 @@ public final class ZNodeClearer {
|
||||
return masterServerName;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true if cluster is configured with master-rs collocation
|
||||
* @deprecated since 2.4.0, will be removed in 3.0.0.
|
||||
* @see <a href="https://issues.apache.org/jira/browse/HBASE-15549">HBASE-15549</a>
|
||||
*/
|
||||
@Deprecated
|
||||
private static boolean tablesOnMaster(Configuration conf) {
|
||||
boolean tablesOnMaster = true;
|
||||
String confValue = conf.get(BaseLoadBalancer.TABLES_ON_MASTER);
|
||||
if (confValue != null && confValue.equalsIgnoreCase("none")) {
|
||||
tablesOnMaster = false;
|
||||
}
|
||||
return tablesOnMaster;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete the master znode if its content (ServerName string) is the same
|
||||
* as the one in the znode file. (env: HBASE_ZNODE_FILE). I case of master-rs
|
||||
@ -185,15 +166,7 @@ public final class ZNodeClearer {
|
||||
String znodeFileContent;
|
||||
try {
|
||||
znodeFileContent = ZNodeClearer.readMyEphemeralNodeOnDisk();
|
||||
if (ZNodeClearer.tablesOnMaster(conf)) {
|
||||
// In case of master crash also remove rsZnode since master is also regionserver
|
||||
ZKUtil.deleteNodeFailSilent(zkw,
|
||||
ZNodePaths.joinZNode(zkw.getZNodePaths().rsZNode, znodeFileContent));
|
||||
return MasterAddressTracker.deleteIfEquals(zkw,
|
||||
ZNodeClearer.parseMasterServerName(znodeFileContent));
|
||||
} else {
|
||||
return MasterAddressTracker.deleteIfEquals(zkw, znodeFileContent);
|
||||
}
|
||||
return MasterAddressTracker.deleteIfEquals(zkw, znodeFileContent);
|
||||
} catch (FileNotFoundException fnfe) {
|
||||
// If no file, just keep going -- return success.
|
||||
LOG.warn("Can't find the znode file; presume non-fatal", fnfe);
|
||||
@ -201,9 +174,6 @@ public final class ZNodeClearer {
|
||||
} catch (IOException e) {
|
||||
LOG.warn("Can't read the content of the znode file", e);
|
||||
return false;
|
||||
} catch (KeeperException e) {
|
||||
LOG.warn("ZooKeeper exception deleting znode", e);
|
||||
return false;
|
||||
} finally {
|
||||
zkw.close();
|
||||
}
|
||||
|
||||
@ -605,17 +605,14 @@ public class HMaster extends HRegionServer implements MasterServices {
|
||||
}
|
||||
|
||||
/**
|
||||
* If configured to put regions on active master,
|
||||
* wait till a backup master becomes active.
|
||||
* Otherwise, loop till the server is stopped or aborted.
|
||||
* Loop till the server is stopped or aborted.
|
||||
*/
|
||||
@Override
|
||||
protected void waitForMasterActive(){
|
||||
protected void waitForMasterActive() {
|
||||
if (maintenanceMode) {
|
||||
return;
|
||||
}
|
||||
boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(conf);
|
||||
while (!(tablesOnMaster && activeMaster) && !isStopped() && !isAborted()) {
|
||||
while (!isStopped() && !isAborted()) {
|
||||
sleeper.sleep();
|
||||
}
|
||||
}
|
||||
@ -658,7 +655,7 @@ public class HMaster extends HRegionServer implements MasterServices {
|
||||
protected void configureInfoServer() {
|
||||
infoServer.addUnprivilegedServlet("master-status", "/master-status", MasterStatusServlet.class);
|
||||
infoServer.setAttribute(MASTER, this);
|
||||
if (LoadBalancer.isTablesOnMaster(conf)) {
|
||||
if (maintenanceMode) {
|
||||
super.configureInfoServer();
|
||||
}
|
||||
}
|
||||
@ -3703,7 +3700,7 @@ public class HMaster extends HRegionServer implements MasterServices {
|
||||
|
||||
@Override
|
||||
public Map<String, ReplicationStatus> getWalGroupsReplicationStatus() {
|
||||
if (!this.isOnline() || !LoadBalancer.isMasterCanHostUserRegions(conf)) {
|
||||
if (!this.isOnline() || !maintenanceMode) {
|
||||
return new HashMap<>();
|
||||
}
|
||||
return super.getWalGroupsReplicationStatus();
|
||||
|
||||
@ -48,25 +48,6 @@ import org.apache.yetus.audience.InterfaceAudience;
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public interface LoadBalancer extends Configurable, Stoppable, ConfigurationObserver {
|
||||
/**
|
||||
* Master can carry regions as of hbase-2.0.0.
|
||||
* By default, it carries no tables.
|
||||
* TODO: Add any | system as flags to indicate what it can do.
|
||||
*
|
||||
* @deprecated since 2.4.0, will be removed in 3.0.0.
|
||||
* @see <a href="https://issues.apache.org/jira/browse/HBASE-15549">HBASE-15549</a>
|
||||
*/
|
||||
@Deprecated
|
||||
String TABLES_ON_MASTER = "hbase.balancer.tablesOnMaster";
|
||||
|
||||
/**
|
||||
* Master carries system tables.
|
||||
*
|
||||
* @deprecated since 2.4.0, will be removed in 3.0.0.
|
||||
* @see <a href="https://issues.apache.org/jira/browse/HBASE-15549">HBASE-15549</a>
|
||||
*/
|
||||
@Deprecated
|
||||
String SYSTEM_TABLES_ON_MASTER = "hbase.balancer.tablesOnMaster.systemTablesOnly";
|
||||
|
||||
// Used to signal to the caller that the region(s) cannot be assigned
|
||||
// We deliberately use 'localhost' so the operation will fail fast
|
||||
@ -164,32 +145,4 @@ public interface LoadBalancer extends Configurable, Stoppable, ConfigurationObse
|
||||
|
||||
/*Updates balancer status tag reported to JMX*/
|
||||
void updateBalancerStatus(boolean status);
|
||||
|
||||
/**
|
||||
* @return true if Master carries regions
|
||||
* @deprecated since 2.4.0, will be removed in 3.0.0.
|
||||
* @see <a href="https://issues.apache.org/jira/browse/HBASE-15549">HBASE-15549</a>
|
||||
*/
|
||||
@Deprecated
|
||||
static boolean isTablesOnMaster(Configuration conf) {
|
||||
return conf.getBoolean(TABLES_ON_MASTER, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated since 2.4.0, will be removed in 3.0.0.
|
||||
* @see <a href="https://issues.apache.org/jira/browse/HBASE-15549">HBASE-15549</a>
|
||||
*/
|
||||
@Deprecated
|
||||
static boolean isSystemTablesOnlyOnMaster(Configuration conf) {
|
||||
return conf.getBoolean(SYSTEM_TABLES_ON_MASTER, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated since 2.4.0, will be removed in 3.0.0.
|
||||
* @see <a href="https://issues.apache.org/jira/browse/HBASE-15549">HBASE-15549</a>
|
||||
*/
|
||||
@Deprecated
|
||||
static boolean isMasterCanHostUserRegions(Configuration conf) {
|
||||
return isTablesOnMaster(conf) && !isSystemTablesOnlyOnMaster(conf);
|
||||
}
|
||||
}
|
||||
|
||||
@ -459,8 +459,7 @@ public class MasterRpcServices extends RSRpcServices implements
|
||||
final String name) throws IOException {
|
||||
final Configuration conf = regionServer.getConfiguration();
|
||||
// RpcServer at HM by default enable ByteBufferPool iff HM having user table region in it
|
||||
boolean reservoirEnabled = conf.getBoolean(ByteBuffAllocator.ALLOCATOR_POOL_ENABLED_KEY,
|
||||
LoadBalancer.isMasterCanHostUserRegions(conf));
|
||||
boolean reservoirEnabled = conf.getBoolean(ByteBuffAllocator.ALLOCATOR_POOL_ENABLED_KEY, false);
|
||||
try {
|
||||
return RpcServerFactory.createRpcServer(server, name, getServices(),
|
||||
bindAddress, // use final bindAddress for this server.
|
||||
|
||||
@ -45,7 +45,6 @@ import org.apache.hadoop.hbase.NotServingRegionException;
|
||||
import org.apache.hadoop.hbase.RegionMetrics;
|
||||
import org.apache.hadoop.hbase.ScheduledChore;
|
||||
import org.apache.hadoop.hbase.ServerMetrics;
|
||||
import org.apache.hadoop.hbase.ServerMetricsBuilder;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.YouAreDeadException;
|
||||
import org.apache.hadoop.hbase.client.AsyncClusterConnection;
|
||||
@ -736,13 +735,6 @@ public class ServerManager {
|
||||
}
|
||||
|
||||
int minimumRequired = 1;
|
||||
if (LoadBalancer.isTablesOnMaster(master.getConfiguration()) &&
|
||||
LoadBalancer.isSystemTablesOnlyOnMaster(master.getConfiguration())) {
|
||||
// If Master is carrying regions it will show up as a 'server', but is not handling user-
|
||||
// space regions, so we need a second server.
|
||||
minimumRequired = 2;
|
||||
}
|
||||
|
||||
int minToStart = this.master.getConfiguration().getInt(WAIT_ON_REGIONSERVERS_MINTOSTART, -1);
|
||||
// Ensure we are never less than minimumRequired else stuff won't work.
|
||||
return Math.max(minToStart, minimumRequired);
|
||||
|
||||
@ -27,7 +27,6 @@ import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.Deque;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
@ -37,7 +36,6 @@ import java.util.Set;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Collectors;
|
||||
import org.apache.commons.lang3.NotImplementedException;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.ClusterMetrics;
|
||||
@ -1034,13 +1032,6 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
||||
protected ServerName masterServerName;
|
||||
protected MasterServices services;
|
||||
|
||||
/**
|
||||
* @deprecated since 2.4.0, will be removed in 3.0.0.
|
||||
* @see <a href="https://issues.apache.org/jira/browse/HBASE-15549">HBASE-15549</a>
|
||||
*/
|
||||
@Deprecated
|
||||
protected boolean onlySystemTablesOnMaster;
|
||||
|
||||
@Override
|
||||
public void setConf(Configuration conf) {
|
||||
this.config = conf;
|
||||
@ -1057,15 +1048,13 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
||||
overallSlop = 1;
|
||||
}
|
||||
|
||||
this.onlySystemTablesOnMaster = LoadBalancer.isSystemTablesOnlyOnMaster(this.config);
|
||||
|
||||
this.rackManager = new RackManager(getConf());
|
||||
if (useRegionFinder) {
|
||||
regionFinder.setConf(conf);
|
||||
}
|
||||
this.isByTable = conf.getBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable);
|
||||
// Print out base configs. Don't print overallSlop since it for simple balancer exclusively.
|
||||
LOG.info("slop={}, systemTablesOnMaster={}", this.slop, this.onlySystemTablesOnMaster);
|
||||
LOG.info("slop={}", this.slop);
|
||||
}
|
||||
|
||||
protected void setSlop(Configuration conf) {
|
||||
@ -1073,95 +1062,6 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
||||
this.overallSlop = conf.getFloat("hbase.regions.overallSlop", slop);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a region belongs to some system table.
|
||||
* If so, the primary replica may be expected to be put on the master regionserver.
|
||||
*
|
||||
* @deprecated since 2.4.0, will be removed in 3.0.0.
|
||||
* @see <a href="https://issues.apache.org/jira/browse/HBASE-15549">HBASE-15549</a>
|
||||
*/
|
||||
@Deprecated
|
||||
public boolean shouldBeOnMaster(RegionInfo region) {
|
||||
return this.onlySystemTablesOnMaster && region.getTable().isSystemTable();
|
||||
}
|
||||
|
||||
/**
|
||||
* Balance the regions that should be on master regionserver.
|
||||
*
|
||||
* @deprecated since 2.4.0, will be removed in 3.0.0.
|
||||
* @see <a href="https://issues.apache.org/jira/browse/HBASE-15549">HBASE-15549</a>
|
||||
*/
|
||||
@Deprecated
|
||||
protected List<RegionPlan> balanceMasterRegions(Map<ServerName, List<RegionInfo>> clusterMap) {
|
||||
if (masterServerName == null || clusterMap == null || clusterMap.size() <= 1) return null;
|
||||
List<RegionPlan> plans = null;
|
||||
List<RegionInfo> regions = clusterMap.get(masterServerName);
|
||||
if (regions != null) {
|
||||
Iterator<ServerName> keyIt = null;
|
||||
for (RegionInfo region: regions) {
|
||||
if (shouldBeOnMaster(region)) continue;
|
||||
|
||||
// Find a non-master regionserver to host the region
|
||||
if (keyIt == null || !keyIt.hasNext()) {
|
||||
keyIt = clusterMap.keySet().iterator();
|
||||
}
|
||||
ServerName dest = keyIt.next();
|
||||
if (masterServerName.equals(dest)) {
|
||||
if (!keyIt.hasNext()) {
|
||||
keyIt = clusterMap.keySet().iterator();
|
||||
}
|
||||
dest = keyIt.next();
|
||||
}
|
||||
|
||||
// Move this region away from the master regionserver
|
||||
RegionPlan plan = new RegionPlan(region, masterServerName, dest);
|
||||
if (plans == null) {
|
||||
plans = new ArrayList<>();
|
||||
}
|
||||
plans.add(plan);
|
||||
}
|
||||
}
|
||||
for (Map.Entry<ServerName, List<RegionInfo>> server: clusterMap.entrySet()) {
|
||||
if (masterServerName.equals(server.getKey())) continue;
|
||||
for (RegionInfo region: server.getValue()) {
|
||||
if (!shouldBeOnMaster(region)) continue;
|
||||
|
||||
// Move this region to the master regionserver
|
||||
RegionPlan plan = new RegionPlan(region, server.getKey(), masterServerName);
|
||||
if (plans == null) {
|
||||
plans = new ArrayList<>();
|
||||
}
|
||||
plans.add(plan);
|
||||
}
|
||||
}
|
||||
return plans;
|
||||
}
|
||||
|
||||
/**
|
||||
* If master is configured to carry system tables only, in here is
|
||||
* where we figure what to assign it.
|
||||
*
|
||||
* @deprecated since 2.4.0, will be removed in 3.0.0.
|
||||
* @see <a href="https://issues.apache.org/jira/browse/HBASE-15549">HBASE-15549</a>
|
||||
*/
|
||||
@Deprecated
|
||||
@NonNull
|
||||
protected Map<ServerName, List<RegionInfo>> assignMasterSystemRegions(
|
||||
Collection<RegionInfo> regions, List<ServerName> servers) {
|
||||
Map<ServerName, List<RegionInfo>> assignments = new TreeMap<>();
|
||||
if (this.onlySystemTablesOnMaster) {
|
||||
if (masterServerName != null && servers.contains(masterServerName)) {
|
||||
assignments.put(masterServerName, new ArrayList<>());
|
||||
for (RegionInfo region : regions) {
|
||||
if (shouldBeOnMaster(region)) {
|
||||
assignments.get(masterServerName).add(region);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return assignments;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Configuration getConf() {
|
||||
return this.config;
|
||||
@ -1281,26 +1181,8 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
||||
@Override
|
||||
@NonNull
|
||||
public Map<ServerName, List<RegionInfo>> roundRobinAssignment(List<RegionInfo> regions,
|
||||
List<ServerName> servers) throws HBaseIOException {
|
||||
List<ServerName> servers) throws HBaseIOException {
|
||||
metricsBalancer.incrMiscInvocations();
|
||||
Map<ServerName, List<RegionInfo>> assignments = assignMasterSystemRegions(regions, servers);
|
||||
if (!assignments.isEmpty()) {
|
||||
servers = new ArrayList<>(servers);
|
||||
// Guarantee not to put other regions on master
|
||||
servers.remove(masterServerName);
|
||||
List<RegionInfo> masterRegions = assignments.get(masterServerName);
|
||||
if (!masterRegions.isEmpty()) {
|
||||
regions = new ArrayList<>(regions);
|
||||
regions.removeAll(masterRegions);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* only need assign system table
|
||||
*/
|
||||
if (regions.isEmpty()) {
|
||||
return assignments;
|
||||
}
|
||||
|
||||
int numServers = servers == null ? 0 : servers.size();
|
||||
if (numServers == 0) {
|
||||
LOG.warn("Wanted to do round robin assignment but no servers to assign to");
|
||||
@ -1313,12 +1195,11 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
||||
// and balanced. This should also run fast with fewer number of iterations.
|
||||
|
||||
if (numServers == 1) { // Only one server, nothing fancy we can do here
|
||||
ServerName server = servers.get(0);
|
||||
assignments.put(server, new ArrayList<>(regions));
|
||||
return assignments;
|
||||
return Collections.singletonMap(servers.get(0), new ArrayList<>(regions));
|
||||
}
|
||||
|
||||
Cluster cluster = createCluster(servers, regions);
|
||||
Map<ServerName, List<RegionInfo>> assignments = new HashMap<>();
|
||||
roundRobinAssignment(cluster, regions, servers, assignments);
|
||||
return assignments;
|
||||
}
|
||||
@ -1374,17 +1255,6 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
||||
public ServerName randomAssignment(RegionInfo regionInfo, List<ServerName> servers)
|
||||
throws HBaseIOException {
|
||||
metricsBalancer.incrMiscInvocations();
|
||||
if (servers != null && servers.contains(masterServerName)) {
|
||||
if (shouldBeOnMaster(regionInfo)) {
|
||||
return masterServerName;
|
||||
}
|
||||
if (!LoadBalancer.isTablesOnMaster(getConf())) {
|
||||
// Guarantee we do not put any regions on master
|
||||
servers = new ArrayList<>(servers);
|
||||
servers.remove(masterServerName);
|
||||
}
|
||||
}
|
||||
|
||||
int numServers = servers == null ? 0 : servers.size();
|
||||
if (numServers == 0) {
|
||||
LOG.warn("Wanted to retain assignment but no servers to assign to");
|
||||
@ -1428,28 +1298,14 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
||||
List<ServerName> servers) throws HBaseIOException {
|
||||
// Update metrics
|
||||
metricsBalancer.incrMiscInvocations();
|
||||
Map<ServerName, List<RegionInfo>> assignments = assignMasterSystemRegions(regions.keySet(), servers);
|
||||
if (!assignments.isEmpty()) {
|
||||
servers = new ArrayList<>(servers);
|
||||
// Guarantee not to put other regions on master
|
||||
servers.remove(masterServerName);
|
||||
List<RegionInfo> masterRegions = assignments.get(masterServerName);
|
||||
regions = regions.entrySet().stream().filter(e -> !masterRegions.contains(e.getKey()))
|
||||
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
|
||||
}
|
||||
if (regions.isEmpty()) {
|
||||
return assignments;
|
||||
}
|
||||
|
||||
int numServers = servers == null ? 0 : servers.size();
|
||||
if (numServers == 0) {
|
||||
LOG.warn("Wanted to do retain assignment but no servers to assign to");
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
if (numServers == 1) { // Only one server, nothing fancy we can do here
|
||||
ServerName server = servers.get(0);
|
||||
assignments.put(server, new ArrayList<>(regions.keySet()));
|
||||
return assignments;
|
||||
return Collections.singletonMap(servers.get(0), new ArrayList<>(regions.keySet()));
|
||||
}
|
||||
|
||||
// Group all of the old assignments by their hostname.
|
||||
@ -1458,6 +1314,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
||||
|
||||
// Group the servers by their hostname. It's possible we have multiple
|
||||
// servers on the same host on different ports.
|
||||
Map<ServerName, List<RegionInfo>> assignments = new HashMap<>();
|
||||
ArrayListMultimap<String, ServerName> serversByHostname = ArrayListMultimap.create();
|
||||
for (ServerName server : servers) {
|
||||
assignments.put(server, new ArrayList<>());
|
||||
|
||||
@ -27,12 +27,13 @@ import edu.umd.cs.findbugs.annotations.NonNull;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.hadoop.hbase.HBaseIOException;
|
||||
import org.apache.hadoop.hbase.ServerMetrics;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
@ -43,7 +44,6 @@ import org.apache.hadoop.hbase.favored.FavoredNodesManager;
|
||||
import org.apache.hadoop.hbase.favored.FavoredNodesPlan;
|
||||
import org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position;
|
||||
import org.apache.hadoop.hbase.favored.FavoredNodesPromoter;
|
||||
import org.apache.hadoop.hbase.master.LoadBalancer;
|
||||
import org.apache.hadoop.hbase.master.MasterServices;
|
||||
import org.apache.hadoop.hbase.master.RegionPlan;
|
||||
import org.apache.hadoop.hbase.util.Pair;
|
||||
@ -114,27 +114,12 @@ public class FavoredStochasticBalancer extends StochasticLoadBalancer implements
|
||||
@NonNull
|
||||
public Map<ServerName, List<RegionInfo>> roundRobinAssignment(List<RegionInfo> regions,
|
||||
List<ServerName> servers) throws HBaseIOException {
|
||||
|
||||
metricsBalancer.incrMiscInvocations();
|
||||
|
||||
Set<RegionInfo> regionSet = Sets.newHashSet(regions);
|
||||
Map<ServerName, List<RegionInfo>> assignmentMap = assignMasterSystemRegions(regions, servers);
|
||||
if (!assignmentMap.isEmpty()) {
|
||||
servers = new ArrayList<>(servers);
|
||||
// Guarantee not to put other regions on master
|
||||
servers.remove(masterServerName);
|
||||
List<RegionInfo> masterRegions = assignmentMap.get(masterServerName);
|
||||
if (!masterRegions.isEmpty()) {
|
||||
for (RegionInfo region: masterRegions) {
|
||||
regionSet.remove(region);
|
||||
}
|
||||
}
|
||||
if (regions.isEmpty()) {
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
if (regionSet.isEmpty()) {
|
||||
return assignmentMap;
|
||||
}
|
||||
|
||||
Set<RegionInfo> regionSet = new HashSet<>(regions);
|
||||
Map<ServerName, List<RegionInfo>> assignmentMap = new HashMap<>();
|
||||
try {
|
||||
FavoredNodeAssignmentHelper helper =
|
||||
new FavoredNodeAssignmentHelper(servers, fnm.getRackManager());
|
||||
@ -311,19 +296,6 @@ public class FavoredStochasticBalancer extends StochasticLoadBalancer implements
|
||||
@Override
|
||||
public ServerName randomAssignment(RegionInfo regionInfo, List<ServerName> servers)
|
||||
throws HBaseIOException {
|
||||
|
||||
if (servers != null && servers.contains(masterServerName)) {
|
||||
if (shouldBeOnMaster(regionInfo)) {
|
||||
metricsBalancer.incrMiscInvocations();
|
||||
return masterServerName;
|
||||
}
|
||||
if (!LoadBalancer.isTablesOnMaster(getConf())) {
|
||||
// Guarantee we do not put any regions on master
|
||||
servers = new ArrayList<>(servers);
|
||||
servers.remove(masterServerName);
|
||||
}
|
||||
}
|
||||
|
||||
ServerName destination = null;
|
||||
if (!FavoredNodesManager.isFavoredNodeApplicable(regionInfo)) {
|
||||
return super.randomAssignment(regionInfo, servers);
|
||||
@ -373,7 +345,6 @@ public class FavoredStochasticBalancer extends StochasticLoadBalancer implements
|
||||
@NonNull
|
||||
public Map<ServerName, List<RegionInfo>> retainAssignment(Map<RegionInfo, ServerName> regions,
|
||||
List<ServerName> servers) throws HBaseIOException {
|
||||
|
||||
Map<ServerName, List<RegionInfo>> assignmentMap = Maps.newHashMap();
|
||||
Map<ServerName, List<RegionInfo>> result = super.retainAssignment(regions, servers);
|
||||
if (result.isEmpty()) {
|
||||
@ -381,12 +352,6 @@ public class FavoredStochasticBalancer extends StochasticLoadBalancer implements
|
||||
return result;
|
||||
}
|
||||
|
||||
// Guarantee not to put other regions on master
|
||||
if (servers != null && servers.contains(masterServerName)) {
|
||||
servers = new ArrayList<>(servers);
|
||||
servers.remove(masterServerName);
|
||||
}
|
||||
|
||||
// Lets check if favored nodes info is in META, if not generate now.
|
||||
FavoredNodeAssignmentHelper helper = new FavoredNodeAssignmentHelper(servers, getConf());
|
||||
helper.initialize();
|
||||
|
||||
@ -27,7 +27,6 @@ import java.util.Map;
|
||||
import java.util.NavigableMap;
|
||||
import java.util.Random;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
@ -38,6 +37,7 @@ import org.apache.hadoop.hbase.util.Pair;
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hbase.thirdparty.com.google.common.collect.MinMaxPriorityQueue;
|
||||
|
||||
/**
|
||||
@ -113,11 +113,8 @@ public class SimpleLoadBalancer extends BaseLoadBalancer {
|
||||
Map<ServerName, Integer> server2LoadMap = new HashMap<>();
|
||||
float sum = 0;
|
||||
for (Map.Entry<TableName, Map<ServerName, List<RegionInfo>>> clusterEntry : clusterLoad
|
||||
.entrySet()) {
|
||||
.entrySet()) {
|
||||
for (Map.Entry<ServerName, List<RegionInfo>> entry : clusterEntry.getValue().entrySet()) {
|
||||
if (entry.getKey().equals(masterServerName)) {
|
||||
continue; // we shouldn't include master as potential assignee
|
||||
}
|
||||
int regionNum = entry.getValue().size();
|
||||
server2LoadMap.compute(entry.getKey(), (k, v) -> v == null ? regionNum : regionNum + v);
|
||||
sum += regionNum;
|
||||
@ -255,18 +252,6 @@ public class SimpleLoadBalancer extends BaseLoadBalancer {
|
||||
@Override
|
||||
public List<RegionPlan> balanceTable(TableName tableName,
|
||||
Map<ServerName, List<RegionInfo>> loadOfOneTable) {
|
||||
List<RegionPlan> regionsToReturn = balanceMasterRegions(loadOfOneTable);
|
||||
if (regionsToReturn != null || loadOfOneTable == null || loadOfOneTable.size() <= 1) {
|
||||
return regionsToReturn;
|
||||
}
|
||||
if (masterServerName != null && loadOfOneTable.containsKey(masterServerName)) {
|
||||
if (loadOfOneTable.size() <= 2) {
|
||||
return null;
|
||||
}
|
||||
loadOfOneTable = new HashMap<>(loadOfOneTable);
|
||||
loadOfOneTable.remove(masterServerName);
|
||||
}
|
||||
|
||||
long startTime = System.currentTimeMillis();
|
||||
|
||||
// construct a Cluster object with clusterMap and rest of the
|
||||
@ -294,7 +279,7 @@ public class SimpleLoadBalancer extends BaseLoadBalancer {
|
||||
// TODO: Look at data block locality or a more complex load to do this
|
||||
MinMaxPriorityQueue<RegionPlan> regionsToMove =
|
||||
MinMaxPriorityQueue.orderedBy(rpComparator).create();
|
||||
regionsToReturn = new ArrayList<>();
|
||||
List<RegionPlan> regionsToReturn = new ArrayList<>();
|
||||
|
||||
// Walk down most loaded, pruning each to the max
|
||||
int serversOverloaded = 0;
|
||||
@ -322,9 +307,6 @@ public class SimpleLoadBalancer extends BaseLoadBalancer {
|
||||
hri = regions.get(regions.size() - 1 - i);
|
||||
}
|
||||
i++;
|
||||
// Don't rebalance special regions.
|
||||
if (shouldBeOnMaster(hri)
|
||||
&& masterServerName.equals(sal.getServerName())) continue;
|
||||
regionsToMove.add(new RegionPlan(hri, sal.getServerName(), null));
|
||||
numTaken++;
|
||||
if (numTaken >= numToOffload) break;
|
||||
|
||||
@ -381,19 +381,6 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
|
||||
@Override
|
||||
public synchronized List<RegionPlan> balanceTable(TableName tableName, Map<ServerName,
|
||||
List<RegionInfo>> loadOfOneTable) {
|
||||
List<RegionPlan> plans = balanceMasterRegions(loadOfOneTable);
|
||||
if (plans != null || loadOfOneTable == null || loadOfOneTable.size() <= 1) {
|
||||
return plans;
|
||||
}
|
||||
|
||||
if (masterServerName != null && loadOfOneTable.containsKey(masterServerName)) {
|
||||
if (loadOfOneTable.size() <= 2) {
|
||||
return null;
|
||||
}
|
||||
loadOfOneTable = new HashMap<>(loadOfOneTable);
|
||||
loadOfOneTable.remove(masterServerName);
|
||||
}
|
||||
|
||||
// On clusters with lots of HFileLinks or lots of reference files,
|
||||
// instantiating the storefile infos can be quite expensive.
|
||||
// Allow turning this feature off if the locality cost is not going to
|
||||
@ -485,7 +472,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
|
||||
// update costs metrics
|
||||
updateStochasticCosts(tableName, curOverallCost, curFunctionCosts);
|
||||
if (initCost > currentCost) {
|
||||
plans = createRegionPlans(cluster);
|
||||
List<RegionPlan> plans = createRegionPlans(cluster);
|
||||
LOG.info("Finished computing new load balance plan. Computation took {}" +
|
||||
" to try {} different iterations. Found a solution that moves " +
|
||||
"{} regions; Going from a computed cost of {}" +
|
||||
|
||||
@ -102,7 +102,6 @@ import org.apache.hadoop.hbase.exceptions.RegionMovedException;
|
||||
import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
|
||||
import org.apache.hadoop.hbase.exceptions.UnknownProtocolException;
|
||||
import org.apache.hadoop.hbase.executor.ExecutorService;
|
||||
import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorConfig;
|
||||
import org.apache.hadoop.hbase.executor.ExecutorType;
|
||||
import org.apache.hadoop.hbase.fs.HFileSystem;
|
||||
import org.apache.hadoop.hbase.http.InfoServer;
|
||||
@ -119,7 +118,6 @@ import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
|
||||
import org.apache.hadoop.hbase.ipc.ServerRpcController;
|
||||
import org.apache.hadoop.hbase.log.HBaseMarkers;
|
||||
import org.apache.hadoop.hbase.master.HMaster;
|
||||
import org.apache.hadoop.hbase.master.LoadBalancer;
|
||||
import org.apache.hadoop.hbase.master.MasterRpcServicesVersionWrapper;
|
||||
import org.apache.hadoop.hbase.master.RegionState;
|
||||
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
|
||||
@ -644,7 +642,7 @@ public class HRegionServer extends Thread implements
|
||||
regionServerAccounting = new RegionServerAccounting(conf);
|
||||
|
||||
boolean isMasterNotCarryTable =
|
||||
this instanceof HMaster && !LoadBalancer.isTablesOnMaster(conf);
|
||||
this instanceof HMaster && !((HMaster) this).isInMaintenanceMode();
|
||||
|
||||
// no need to instantiate block cache and mob file cache when master not carry table
|
||||
if (!isMasterNotCarryTable) {
|
||||
@ -1944,11 +1942,10 @@ public class HRegionServer extends Thread implements
|
||||
* be hooked up to WAL.
|
||||
*/
|
||||
private void setupWALAndReplication() throws IOException {
|
||||
boolean isMasterNoTableOrSystemTableOnly = this instanceof HMaster &&
|
||||
!LoadBalancer.isMasterCanHostUserRegions(conf);
|
||||
boolean isMaster = this instanceof HMaster;
|
||||
WALFactory factory =
|
||||
new WALFactory(conf, serverName.toString(), this, !isMasterNoTableOrSystemTableOnly);
|
||||
if (!isMasterNoTableOrSystemTableOnly) {
|
||||
new WALFactory(conf, serverName.toString(), this, !isMaster);
|
||||
if (!isMaster) {
|
||||
// TODO Replication make assumptions here based on the default filesystem impl
|
||||
Path oldLogDir = new Path(walRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
|
||||
String logName = AbstractFSWALProvider.getWALDirectoryName(this.serverName.toString());
|
||||
|
||||
@ -39,7 +39,6 @@ import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.TableExistsException;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.TableNotFoundException;
|
||||
import org.apache.hadoop.hbase.master.LoadBalancer;
|
||||
import org.apache.hadoop.hbase.testclassification.ClientTests;
|
||||
import org.apache.hadoop.hbase.testclassification.LargeTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
@ -368,19 +367,11 @@ public class TestAdmin extends TestAdminBase {
|
||||
}
|
||||
regs.add(loc.getRegion());
|
||||
}
|
||||
boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
|
||||
if (tablesOnMaster) {
|
||||
// Ignore the master region server,
|
||||
// which contains less regions by intention.
|
||||
numRS--;
|
||||
}
|
||||
float average = (float) expectedRegions / numRS;
|
||||
int min = (int) Math.floor(average);
|
||||
int max = (int) Math.ceil(average);
|
||||
for (List<RegionInfo> regionList : server2Regions.values()) {
|
||||
assertTrue(
|
||||
"numRS=" + numRS + ", min=" + min + ", max=" + max + ", size=" + regionList.size() +
|
||||
", tablesOnMaster=" + tablesOnMaster,
|
||||
assertTrue("numRS=" + numRS + ", min=" + min + ", max=" + max + ", size=" + regionList.size(),
|
||||
regionList.size() == min || regionList.size() == max);
|
||||
}
|
||||
}
|
||||
|
||||
@ -22,22 +22,18 @@ import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.CompletionException;
|
||||
import org.apache.hadoop.hbase.ClientMetaTableAccessor;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.HRegionLocation;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.TableExistsException;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.TableNotFoundException;
|
||||
import org.apache.hadoop.hbase.master.LoadBalancer;
|
||||
import org.apache.hadoop.hbase.testclassification.ClientTests;
|
||||
import org.apache.hadoop.hbase.testclassification.LargeTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
@ -125,7 +121,6 @@ public class TestAsyncTableAdminApi extends TestAsyncAdminBase {
|
||||
new byte[] { 4, 4, 4 }, new byte[] { 5, 5, 5 }, new byte[] { 6, 6, 6 },
|
||||
new byte[] { 7, 7, 7 }, new byte[] { 8, 8, 8 }, new byte[] { 9, 9, 9 }, };
|
||||
int expectedRegions = splitKeys.length + 1;
|
||||
boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
|
||||
createTableWithDefaultConf(tableName, splitKeys);
|
||||
|
||||
boolean tableAvailable = admin.isTableAvailable(tableName).get();
|
||||
@ -173,9 +168,6 @@ public class TestAsyncTableAdminApi extends TestAsyncAdminBase {
|
||||
hri = hris.next().getRegion();
|
||||
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[8]));
|
||||
assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
|
||||
if (tablesOnMaster) {
|
||||
verifyRoundRobinDistribution(regions, expectedRegions);
|
||||
}
|
||||
|
||||
// Now test using start/end with a number of regions
|
||||
|
||||
@ -228,10 +220,6 @@ public class TestAsyncTableAdminApi extends TestAsyncAdminBase {
|
||||
hri = hris.next().getRegion();
|
||||
assertTrue(Bytes.equals(hri.getStartKey(), new byte[] { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }));
|
||||
assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
|
||||
if (tablesOnMaster) {
|
||||
// This don't work if master is not carrying regions. FIX. TODO.
|
||||
verifyRoundRobinDistribution(regions, expectedRegions);
|
||||
}
|
||||
|
||||
// Try once more with something that divides into something infinite
|
||||
startKey = new byte[] { 0, 0, 0, 0, 0, 0 };
|
||||
@ -249,10 +237,6 @@ public class TestAsyncTableAdminApi extends TestAsyncAdminBase {
|
||||
"Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(),
|
||||
expectedRegions, regions.size());
|
||||
System.err.println("Found " + regions.size() + " regions");
|
||||
if (tablesOnMaster) {
|
||||
// This don't work if master is not carrying regions. FIX. TODO.
|
||||
verifyRoundRobinDistribution(regions, expectedRegions);
|
||||
}
|
||||
|
||||
// Try an invalid case where there are duplicate split keys
|
||||
splitKeys = new byte[][] { new byte[] { 1, 1, 1 }, new byte[] { 2, 2, 2 },
|
||||
@ -266,27 +250,6 @@ public class TestAsyncTableAdminApi extends TestAsyncAdminBase {
|
||||
}
|
||||
}
|
||||
|
||||
private void verifyRoundRobinDistribution(List<HRegionLocation> regions, int expectedRegions) {
|
||||
int numRS = TEST_UTIL.getMiniHBaseCluster().getNumLiveRegionServers();
|
||||
|
||||
Map<ServerName, List<RegionInfo>> server2Regions = new HashMap<>();
|
||||
regions.stream().forEach((loc) -> {
|
||||
ServerName server = loc.getServerName();
|
||||
server2Regions.computeIfAbsent(server, (s) -> new ArrayList<>()).add(loc.getRegion());
|
||||
});
|
||||
if (numRS >= 2) {
|
||||
// Ignore the master region server,
|
||||
// which contains less regions by intention.
|
||||
numRS--;
|
||||
}
|
||||
float average = (float) expectedRegions / numRS;
|
||||
int min = (int) Math.floor(average);
|
||||
int max = (int) Math.ceil(average);
|
||||
server2Regions.values().forEach((regionList) -> {
|
||||
assertTrue(regionList.size() == min || regionList.size() == max);
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateTableWithOnlyEmptyStartRow() throws Exception {
|
||||
byte[][] splitKeys = new byte[1][];
|
||||
|
||||
@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.client;
|
||||
|
||||
import static org.apache.hadoop.hbase.HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT;
|
||||
import static org.apache.hadoop.hbase.io.ByteBuffAllocator.MAX_BUFFER_COUNT_KEY;
|
||||
import static org.apache.hadoop.hbase.master.LoadBalancer.TABLES_ON_MASTER;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import java.io.IOException;
|
||||
@ -49,7 +48,6 @@ import org.apache.hadoop.hbase.testclassification.LargeTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.RetryCounter;
|
||||
import org.apache.hadoop.hbase.util.Threads;
|
||||
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
@ -60,6 +58,7 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
|
||||
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
|
||||
|
||||
/**
|
||||
* Will split the table, and move region randomly when testing.
|
||||
@ -92,7 +91,6 @@ public class TestAsyncTableGetMultiThreaded {
|
||||
}
|
||||
|
||||
protected static void setUp(MemoryCompactionPolicy memoryCompaction) throws Exception {
|
||||
TEST_UTIL.getConfiguration().set(TABLES_ON_MASTER, "none");
|
||||
TEST_UTIL.getConfiguration().setLong(HBASE_CLIENT_META_OPERATION_TIMEOUT, 60000L);
|
||||
TEST_UTIL.getConfiguration().setInt(MAX_BUFFER_COUNT_KEY, 100);
|
||||
TEST_UTIL.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY,
|
||||
|
||||
@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.MiniHBaseCluster;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.TableNameTestRule;
|
||||
import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
|
||||
import org.apache.hadoop.hbase.master.LoadBalancer;
|
||||
import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
|
||||
import org.apache.hadoop.hbase.testclassification.ClientTests;
|
||||
import org.apache.hadoop.hbase.testclassification.LargeTests;
|
||||
@ -1252,12 +1251,11 @@ public class TestFromClientSide4 extends FromClientSideBase {
|
||||
|
||||
// test that the same unmanaged connection works with a new
|
||||
// Admin and can connect to the new master;
|
||||
boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
|
||||
try (Admin admin = conn.getAdmin()) {
|
||||
assertTrue(admin.tableExists(tableName));
|
||||
assertEquals(
|
||||
admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().size(),
|
||||
SLAVES + (tablesOnMaster ? 1 : 0));
|
||||
SLAVES);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.favored.FavoredNodeAssignmentHelper;
|
||||
import org.apache.hadoop.hbase.favored.FavoredNodesManager;
|
||||
import org.apache.hadoop.hbase.master.LoadBalancer;
|
||||
import org.apache.hadoop.hbase.master.ServerManager;
|
||||
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
|
||||
import org.apache.hadoop.hbase.master.balancer.LoadOnlyFavoredStochasticBalancer;
|
||||
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
|
||||
import org.apache.hadoop.hbase.regionserver.HRegionServer;
|
||||
@ -96,8 +95,6 @@ public class TestTableFavoredNodes {
|
||||
LoadOnlyFavoredStochasticBalancer.class, LoadBalancer.class);
|
||||
conf.set(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, "" + SLAVES);
|
||||
|
||||
// This helps test if RS get the appropriate FN updates.
|
||||
conf.set(BaseLoadBalancer.TABLES_ON_MASTER, "none");
|
||||
TEST_UTIL.startMiniCluster(SLAVES);
|
||||
TEST_UTIL.getMiniHBaseCluster().waitForActiveAndReadyMaster(WAIT_TIMEOUT);
|
||||
}
|
||||
|
||||
@ -34,7 +34,6 @@ import org.apache.hadoop.hbase.MiniHBaseCluster;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.client.Put;
|
||||
import org.apache.hadoop.hbase.client.Table;
|
||||
import org.apache.hadoop.hbase.master.LoadBalancer;
|
||||
import org.apache.hadoop.hbase.regionserver.HRegion;
|
||||
import org.apache.hadoop.hbase.regionserver.HRegionServer;
|
||||
import org.apache.hadoop.hbase.regionserver.Region;
|
||||
@ -118,8 +117,7 @@ public class TestBlockReorderMultiBlocks {
|
||||
|
||||
MiniHBaseCluster hbm = htu.startMiniHBaseCluster();
|
||||
hbm.waitForActiveAndReadyMaster();
|
||||
HRegionServer targetRs = LoadBalancer.isTablesOnMaster(hbm.getConf())? hbm.getMaster():
|
||||
hbm.getRegionServer(0);
|
||||
HRegionServer targetRs = hbm.getRegionServer(0);
|
||||
|
||||
// We want to have a datanode with the same name as the region server, so
|
||||
// we're going to get the regionservername, and start a new datanode with this name.
|
||||
|
||||
@ -122,8 +122,7 @@ public class TestMasterMetrics {
|
||||
@Test
|
||||
public void testDefaultMasterMetrics() throws Exception {
|
||||
MetricsMasterSource masterSource = master.getMasterMetrics().getMetricsSource();
|
||||
boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
|
||||
metricsHelper.assertGauge("numRegionServers", 1 + (tablesOnMaster ? 1 : 0), masterSource);
|
||||
metricsHelper.assertGauge("numRegionServers", 1, masterSource);
|
||||
metricsHelper.assertGauge("averageLoad", 1, masterSource);
|
||||
metricsHelper.assertGauge("numDeadRegionServers", 0, masterSource);
|
||||
metricsHelper.assertGauge("numDrainingRegionServers", 0, masterSource);
|
||||
|
||||
@ -82,8 +82,7 @@ public class TestMasterMetricsWrapper {
|
||||
assertEquals(master.getMasterStartTime(), info.getStartTime());
|
||||
assertEquals(master.getMasterCoprocessors().length, info.getCoprocessors().length);
|
||||
assertEquals(master.getServerManager().getOnlineServersList().size(), info.getNumRegionServers());
|
||||
int regionServerCount =
|
||||
NUM_RS + (LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration())? 1: 0);
|
||||
int regionServerCount = NUM_RS;
|
||||
assertEquals(regionServerCount, info.getNumRegionServers());
|
||||
|
||||
String zkServers = info.getZookeeperQuorum();
|
||||
|
||||
@ -121,11 +121,10 @@ public class TestMasterNoCluster {
|
||||
* Test starting master then stopping it before its fully up.
|
||||
*/
|
||||
@Test
|
||||
public void testStopDuringStart()
|
||||
throws IOException, KeeperException, InterruptedException {
|
||||
public void testStopDuringStart() throws IOException, KeeperException, InterruptedException {
|
||||
HMaster master = new HMaster(TESTUTIL.getConfiguration());
|
||||
master.start();
|
||||
// Immediately have it stop. We used hang in assigning meta.
|
||||
// Immediately have it stop. We used hang in assigning meta.
|
||||
master.stopMaster();
|
||||
master.join();
|
||||
}
|
||||
@ -150,9 +149,10 @@ public class TestMasterNoCluster {
|
||||
conf.set(HConstants.CLIENT_ZOOKEEPER_QUORUM, HConstants.LOCALHOST);
|
||||
conf.setInt(HConstants.CLIENT_ZOOKEEPER_CLIENT_PORT,
|
||||
TESTUTIL.getZkCluster().getClientPort() + 1);
|
||||
// need to enable maintenance mode so we will start master as a region server
|
||||
conf.setBoolean(HMaster.MAINTENANCE_MODE, true);
|
||||
// settings to allow us not to start additional RS
|
||||
conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
|
||||
conf.setBoolean(LoadBalancer.TABLES_ON_MASTER, true);
|
||||
// main setting for this test case
|
||||
conf.setBoolean(HConstants.CLIENT_ZOOKEEPER_OBSERVER_MODE, true);
|
||||
HMaster master = new HMaster(conf);
|
||||
|
||||
@ -67,13 +67,6 @@ public class TestMasterNotCarryTable {
|
||||
UTIL.shutdownMiniZKCluster();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMasterNotCarryTable() {
|
||||
// The default config is false
|
||||
assertFalse(LoadBalancer.isTablesOnMaster(UTIL.getConfiguration()));
|
||||
assertFalse(LoadBalancer.isSystemTablesOnlyOnMaster(UTIL.getConfiguration()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMasterBlockCache() {
|
||||
// no need to instantiate block cache.
|
||||
|
||||
@ -157,10 +157,6 @@ public class TestBaseLoadBalancer extends BalancerTestBase {
|
||||
hris.add(RegionInfoBuilder.FIRST_META_REGIONINFO);
|
||||
tmp.add(master);
|
||||
Map<ServerName, List<RegionInfo>> plans = loadBalancer.roundRobinAssignment(hris, tmp);
|
||||
if (LoadBalancer.isTablesOnMaster(loadBalancer.getConf())) {
|
||||
assertTrue(plans.get(master).contains(RegionInfoBuilder.FIRST_META_REGIONINFO));
|
||||
assertEquals(1, plans.get(master).size());
|
||||
}
|
||||
int totalRegion = 0;
|
||||
for (List<RegionInfo> regions: plans.values()) {
|
||||
totalRegion += regions.size();
|
||||
@ -244,12 +240,7 @@ public class TestBaseLoadBalancer extends BalancerTestBase {
|
||||
List<ServerName> allServers = new ArrayList<>(idleServers.size() + 1);
|
||||
allServers.add(ServerName.valueOf("server-" + numberOfIdleServers, 1000, 1L));
|
||||
allServers.addAll(idleServers);
|
||||
LoadBalancer balancer = new MockBalancer() {
|
||||
@Override
|
||||
public boolean shouldBeOnMaster(RegionInfo region) {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
LoadBalancer balancer = new MockBalancer();
|
||||
Configuration conf = HBaseConfiguration.create();
|
||||
conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
|
||||
balancer.setConf(conf);
|
||||
|
||||
@ -99,7 +99,6 @@ public class TestFavoredStochasticBalancerPickers extends BalancerTestBase {
|
||||
conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 30000);
|
||||
conf.setInt("hbase.master.balancer.stochastic.moveCost", 0);
|
||||
conf.setBoolean("hbase.master.balancer.stochastic.execute.maxSteps", true);
|
||||
conf.set(BaseLoadBalancer.TABLES_ON_MASTER, "none");
|
||||
}
|
||||
|
||||
@Before
|
||||
|
||||
@ -1,220 +0,0 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.master.balancer;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.util.List;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.MiniHBaseCluster;
|
||||
import org.apache.hadoop.hbase.StartMiniClusterOption;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.client.Table;
|
||||
import org.apache.hadoop.hbase.master.HMaster;
|
||||
import org.apache.hadoop.hbase.master.LoadBalancer;
|
||||
import org.apache.hadoop.hbase.regionserver.HRegion;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.util.JVMClusterUtil;
|
||||
import org.apache.hadoop.hbase.util.Threads;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
import org.junit.rules.TestName;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Test options for regions on master; none, system, or any (i.e. master is like any other
|
||||
* regionserver). Checks how regions are deployed when each of the options are enabled.
|
||||
* It then does kill combinations to make sure the distribution is more than just for startup.
|
||||
* NOTE: Regions on Master does not work well. See HBASE-19828. Until addressed, disabling this
|
||||
* test.
|
||||
*/
|
||||
@Ignore
|
||||
@Category({MediumTests.class})
|
||||
public class TestRegionsOnMasterOptions {
|
||||
|
||||
@ClassRule
|
||||
public static final HBaseClassTestRule CLASS_RULE =
|
||||
HBaseClassTestRule.forClass(TestRegionsOnMasterOptions.class);
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(TestRegionsOnMasterOptions.class);
|
||||
@Rule public TestName name = new TestName();
|
||||
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
|
||||
private Configuration c;
|
||||
private String tablesOnMasterOldValue;
|
||||
private String systemTablesOnMasterOldValue;
|
||||
private static final int SLAVES = 3;
|
||||
private static final int MASTERS = 2;
|
||||
// Make the count of REGIONS high enough so I can distingush case where master is only carrying
|
||||
// system regions from the case where it is carrying any region; i.e. 2 system regions vs more
|
||||
// if user + system.
|
||||
private static final int REGIONS = 12;
|
||||
private static final int SYSTEM_REGIONS = 2; // ns and meta -- no acl unless enabled.
|
||||
|
||||
@Before
|
||||
public void setup() {
|
||||
this.c = TEST_UTIL.getConfiguration();
|
||||
this.tablesOnMasterOldValue = c.get(LoadBalancer.TABLES_ON_MASTER);
|
||||
this.systemTablesOnMasterOldValue = c.get(LoadBalancer.SYSTEM_TABLES_ON_MASTER);
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() {
|
||||
unset(LoadBalancer.TABLES_ON_MASTER, this.tablesOnMasterOldValue);
|
||||
unset(LoadBalancer.SYSTEM_TABLES_ON_MASTER, this.systemTablesOnMasterOldValue);
|
||||
}
|
||||
|
||||
private void unset(final String key, final String value) {
|
||||
if (value == null) {
|
||||
c.unset(key);
|
||||
} else {
|
||||
c.set(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRegionsOnAllServers() throws Exception {
|
||||
c.setBoolean(LoadBalancer.TABLES_ON_MASTER, true);
|
||||
c.setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, false);
|
||||
int rsCount = (REGIONS + SYSTEM_REGIONS)/(SLAVES + 1/*Master*/);
|
||||
checkBalance(rsCount, rsCount);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNoRegionOnMaster() throws Exception {
|
||||
c.setBoolean(LoadBalancer.TABLES_ON_MASTER, false);
|
||||
c.setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, false);
|
||||
int rsCount = (REGIONS + SYSTEM_REGIONS)/SLAVES;
|
||||
checkBalance(0, rsCount);
|
||||
}
|
||||
|
||||
@Ignore // Fix this. The Master startup doesn't allow Master reporting as a RegionServer, not
|
||||
// until way late after the Master startup finishes. Needs more work.
|
||||
@Test
|
||||
public void testSystemTablesOnMaster() throws Exception {
|
||||
c.setBoolean(LoadBalancer.TABLES_ON_MASTER, true);
|
||||
c.setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, true);
|
||||
// IS THIS SHORT-CIRCUIT RPC? Yes. Here is how it looks currently if I have an exception
|
||||
// thrown in doBatchMutate inside a Region.
|
||||
//
|
||||
// java.lang.Exception
|
||||
// at org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:3845)
|
||||
// at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:2972)
|
||||
// at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2751)
|
||||
// at org.apache.hadoop.hbase.client.ClientServiceCallable.doMutate(ClientServiceCallable.java:55)
|
||||
// at org.apache.hadoop.hbase.client.HTable$3.rpcCall(HTable.java:585)
|
||||
// at org.apache.hadoop.hbase.client.HTable$3.rpcCall(HTable.java:579)
|
||||
// at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:126)
|
||||
// at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:106)
|
||||
// at org.apache.hadoop.hbase.client.HTable.put(HTable.java:589)
|
||||
// at org.apache.hadoop.hbase.master.TableNamespaceManager.insertIntoNSTable(TableNamespaceManager.java:156)
|
||||
// at org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure.insertIntoNSTable(CreateNamespaceProcedure.java:222)
|
||||
// at org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure.executeFromState(CreateNamespaceProcedure.java:76)
|
||||
// at org.apache.hadoop.hbase.master.procedure.CreateNamespaceProcedure.executeFromState(CreateNamespaceProcedure.java:40)
|
||||
// at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:181)
|
||||
// at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:847)
|
||||
// at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1440)
|
||||
// at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1209)
|
||||
// at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$800(ProcedureExecutor.java:79)
|
||||
// at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1719)
|
||||
//
|
||||
// If I comment out the ConnectionUtils ConnectionImplementation content, I see this:
|
||||
//
|
||||
// java.lang.Exception
|
||||
// at org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:3845)
|
||||
// at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:2972)
|
||||
// at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2751)
|
||||
// at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:41546)
|
||||
// at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:406)
|
||||
// at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:133)
|
||||
// at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:278)
|
||||
// at org.apache.hadoop.hbase.ipc.RpcExecutor$Handler.run(RpcExecutor.java:258)
|
||||
|
||||
checkBalance(SYSTEM_REGIONS, REGIONS/SLAVES);
|
||||
}
|
||||
|
||||
private void checkBalance(int masterCount, int rsCount) throws Exception {
|
||||
StartMiniClusterOption option = StartMiniClusterOption.builder()
|
||||
.numMasters(MASTERS).numRegionServers(SLAVES).numDataNodes(SLAVES).build();
|
||||
MiniHBaseCluster cluster = TEST_UTIL.startMiniCluster(option);
|
||||
TableName tn = TableName.valueOf(this.name.getMethodName());
|
||||
try {
|
||||
Table t = TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY, REGIONS);
|
||||
LOG.info("Server: " + cluster.getMaster().getServerManager().getOnlineServersList());
|
||||
List<HRegion> regions = cluster.getMaster().getRegions();
|
||||
int mActualCount = regions.size();
|
||||
if (masterCount == 0 || masterCount == SYSTEM_REGIONS) {
|
||||
// 0 means no regions on master.
|
||||
assertEquals(masterCount, mActualCount);
|
||||
} else {
|
||||
// This is master as a regionserver scenario.
|
||||
checkCount(masterCount, mActualCount);
|
||||
}
|
||||
// Allow that balance is not exact. FYI, getRegionServerThreads does not include master
|
||||
// thread though it is a regionserver so we have to check master and then below the
|
||||
// regionservers.
|
||||
for (JVMClusterUtil.RegionServerThread rst: cluster.getRegionServerThreads()) {
|
||||
regions = rst.getRegionServer().getRegions();
|
||||
int rsActualCount = regions.size();
|
||||
checkCount(rsActualCount, rsCount);
|
||||
}
|
||||
HMaster oldMaster = cluster.getMaster();
|
||||
cluster.killMaster(oldMaster.getServerName());
|
||||
oldMaster.join();
|
||||
while (cluster.getMaster() == null ||
|
||||
cluster.getMaster().getServerName().equals(oldMaster.getServerName())) {
|
||||
Threads.sleep(10);
|
||||
}
|
||||
while (!cluster.getMaster().isInitialized()) {
|
||||
Threads.sleep(10);
|
||||
}
|
||||
while (cluster.getMaster().getAssignmentManager().
|
||||
computeRegionInTransitionStat().getTotalRITs() > 0) {
|
||||
Threads.sleep(100);
|
||||
LOG.info("Waiting on RIT to go to zero before calling balancer...");
|
||||
}
|
||||
LOG.info("Cluster is up; running balancer");
|
||||
cluster.getMaster().balance();
|
||||
regions = cluster.getMaster().getRegions();
|
||||
int mNewActualCount = regions.size();
|
||||
if (masterCount == 0 || masterCount == SYSTEM_REGIONS) {
|
||||
// 0 means no regions on master. After crash, should still be no regions on master.
|
||||
// If masterCount == SYSTEM_REGIONS, means master only carrying system regions and should
|
||||
// still only carry system regions post crash.
|
||||
assertEquals(masterCount, mNewActualCount);
|
||||
}
|
||||
} finally {
|
||||
LOG.info("Running shutdown of cluster");
|
||||
TEST_UTIL.shutdownMiniCluster();
|
||||
}
|
||||
}
|
||||
|
||||
private void checkCount(int actual, int expected) {
|
||||
assertTrue("Actual=" + actual + ", expected=" + expected,
|
||||
actual >= (expected - 2) && actual <= (expected + 2)); // Lots of slop +/- 2
|
||||
}
|
||||
}
|
||||
@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.Waiter;
|
||||
import org.apache.hadoop.hbase.client.RegionInfo;
|
||||
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
|
||||
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
|
||||
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
@ -58,7 +57,6 @@ public class TestSafemodeBringsDownMaster {
|
||||
|
||||
private static void setupConf(Configuration conf) {
|
||||
conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
|
||||
conf.set(BaseLoadBalancer.TABLES_ON_MASTER, "none");
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
|
||||
@ -30,7 +30,6 @@ import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.StartMiniClusterOption;
|
||||
import org.apache.hadoop.hbase.master.HMaster;
|
||||
import org.apache.hadoop.hbase.master.LoadBalancer;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
|
||||
import org.apache.hadoop.hbase.util.CommonFSUtils;
|
||||
@ -114,8 +113,7 @@ public class TestClusterId {
|
||||
}
|
||||
TEST_UTIL.startMiniHBaseCluster();
|
||||
HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
|
||||
int expected = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration())? 2: 1;
|
||||
assertEquals(expected, master.getServerManager().getOnlineServersList().size());
|
||||
assertEquals(1, master.getServerManager().getOnlineServersList().size());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -34,7 +34,6 @@ import org.apache.hadoop.hbase.MiniHBaseCluster;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.client.RegionInfo;
|
||||
import org.apache.hadoop.hbase.master.HMaster;
|
||||
import org.apache.hadoop.hbase.master.LoadBalancer;
|
||||
import org.apache.hadoop.hbase.master.ServerListener;
|
||||
import org.apache.hadoop.hbase.master.ServerManager;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
@ -104,7 +103,7 @@ public class TestRSKilledWhenInitializing {
|
||||
cluster.getRegionServers().get(i).start();
|
||||
}
|
||||
// Expected total regionservers depends on whether Master can host regions or not.
|
||||
int expectedTotalRegionServers = NUM_RS + (LoadBalancer.isTablesOnMaster(conf)? 1: 0);
|
||||
int expectedTotalRegionServers = NUM_RS;
|
||||
List<ServerName> onlineServersList = null;
|
||||
do {
|
||||
onlineServersList = master.getMaster().getServerManager().getOnlineServersList();
|
||||
|
||||
@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.StartMiniClusterOption;
|
||||
import org.apache.hadoop.hbase.master.LoadBalancer;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
|
||||
import org.apache.hadoop.hbase.util.DNS;
|
||||
@ -113,9 +112,7 @@ public class TestRegionServerHostname {
|
||||
try {
|
||||
ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher();
|
||||
List<String> servers = ZKUtil.listChildrenNoWatch(zkw, zkw.getZNodePaths().rsZNode);
|
||||
// there would be NUM_RS+1 children - one for the master
|
||||
assertTrue(servers.size() ==
|
||||
NUM_RS + (LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration())? 1: 0));
|
||||
assertEquals(NUM_RS, servers.size());
|
||||
for (String server : servers) {
|
||||
assertTrue("From zookeeper: " + server + " hostname: " + hostName,
|
||||
server.startsWith(hostName.toLowerCase(Locale.ROOT)+","));
|
||||
@ -197,8 +194,7 @@ public class TestRegionServerHostname {
|
||||
StartMiniClusterOption option = StartMiniClusterOption.builder()
|
||||
.numMasters(NUM_MASTERS).numRegionServers(NUM_RS).numDataNodes(NUM_RS).build();
|
||||
TEST_UTIL.startMiniCluster(option);
|
||||
boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
|
||||
int expectedRS = NUM_RS + (tablesOnMaster? 1: 0);
|
||||
int expectedRS = NUM_RS;
|
||||
try (ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher()) {
|
||||
List<String> servers = ZKUtil.listChildrenNoWatch(zkw, zkw.getZNodePaths().rsZNode);
|
||||
assertEquals(expectedRS, servers.size());
|
||||
|
||||
@ -18,7 +18,6 @@
|
||||
package org.apache.hadoop.hbase.regionserver;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
@ -27,7 +26,6 @@ import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.CompatibilityFactory;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
@ -55,7 +53,6 @@ import org.apache.hadoop.hbase.client.Scan.ReadType;
|
||||
import org.apache.hadoop.hbase.client.Table;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptor;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.master.LoadBalancer;
|
||||
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
|
||||
import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
|
||||
import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
|
||||
@ -105,13 +102,11 @@ public class TestRegionServerMetrics {
|
||||
private static byte[] qualifier = Bytes.toBytes("qual");
|
||||
private static byte[] val = Bytes.toBytes("val");
|
||||
private static Admin admin;
|
||||
private static boolean TABLES_ON_MASTER;
|
||||
|
||||
@BeforeClass
|
||||
public static void startCluster() throws Exception {
|
||||
metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
|
||||
TEST_UTIL = new HBaseTestingUtility();
|
||||
TABLES_ON_MASTER = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
|
||||
conf = TEST_UTIL.getConfiguration();
|
||||
conf.getLong("hbase.splitlog.max.resubmit", 0);
|
||||
// Make the failure test faster
|
||||
@ -232,7 +227,7 @@ public class TestRegionServerMetrics {
|
||||
|
||||
@Test
|
||||
public void testRegionCount() throws Exception {
|
||||
metricsHelper.assertGauge("regionCount", TABLES_ON_MASTER ? 1 : 2, serverSource);
|
||||
metricsHelper.assertGauge("regionCount", 2, serverSource);
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -274,11 +269,6 @@ public class TestRegionServerMetrics {
|
||||
doNGets(10, true); // true = batch
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
if (TABLES_ON_MASTER) {
|
||||
assertCounter("totalRequestCount", requests + 41);
|
||||
assertCounter("totalRowActionRequestCount", rowActionRequests + 50);
|
||||
assertCounter("readRequestCount", readRequests + 20);
|
||||
}
|
||||
|
||||
|
||||
assertCounter("writeRequestCount", writeRequests + 30);
|
||||
@ -286,30 +276,15 @@ public class TestRegionServerMetrics {
|
||||
doNPuts(30, true);
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
if (TABLES_ON_MASTER) {
|
||||
assertCounter("totalRequestCount", requests + 42);
|
||||
assertCounter("totalRowActionRequestCount", rowActionRequests + 80);
|
||||
assertCounter("readRequestCount", readRequests + 20);
|
||||
}
|
||||
assertCounter("writeRequestCount", writeRequests + 60);
|
||||
|
||||
doScan(10, false); // test after batch put so we have enough lines
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
if (TABLES_ON_MASTER) {
|
||||
assertCounter("totalRequestCount", requests + 52);
|
||||
assertCounter("totalRowActionRequestCount", rowActionRequests + 90);
|
||||
assertCounter("readRequestCount", readRequests + 30);
|
||||
}
|
||||
assertCounter("writeRequestCount", writeRequests + 60);
|
||||
numScanNext += 10;
|
||||
|
||||
doScan(10, true); // true = caching
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
if (TABLES_ON_MASTER) {
|
||||
assertCounter("totalRequestCount", requests + 53);
|
||||
assertCounter("totalRowActionRequestCount", rowActionRequests + 100);
|
||||
assertCounter("readRequestCount", readRequests + 40);
|
||||
}
|
||||
assertCounter("writeRequestCount", writeRequests + 60);
|
||||
numScanNext += 1;
|
||||
}
|
||||
@ -342,7 +317,7 @@ public class TestRegionServerMetrics {
|
||||
TEST_UTIL.getAdmin().flush(tableName);
|
||||
|
||||
metricsRegionServer.getRegionServerWrapper().forceRecompute();
|
||||
assertGauge("storeCount", TABLES_ON_MASTER ? 1 : 5);
|
||||
assertGauge("storeCount", 5);
|
||||
assertGauge("storeFileCount", 1);
|
||||
}
|
||||
|
||||
@ -423,9 +398,6 @@ public class TestRegionServerMetrics {
|
||||
}
|
||||
numScanNext += NUM_SCAN_NEXT;
|
||||
assertRegionMetrics("scanCount", NUM_SCAN_NEXT);
|
||||
if (TABLES_ON_MASTER) {
|
||||
assertCounter("ScanSize_num_ops", numScanNext);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -443,9 +415,6 @@ public class TestRegionServerMetrics {
|
||||
}
|
||||
numScanNext += NUM_SCAN_NEXT;
|
||||
assertRegionMetrics("scanCount", NUM_SCAN_NEXT);
|
||||
if (TABLES_ON_MASTER) {
|
||||
assertCounter("ScanTime_num_ops", numScanNext);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -457,17 +426,11 @@ public class TestRegionServerMetrics {
|
||||
for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) {
|
||||
Result result = resultScanners.next();
|
||||
assertNotNull(result);
|
||||
if (TABLES_ON_MASTER) {
|
||||
assertEquals(1, result.size());
|
||||
}
|
||||
}
|
||||
assertNull(resultScanners.next());
|
||||
}
|
||||
numScanNext += NUM_SCAN_NEXT;
|
||||
assertRegionMetrics("scanCount", NUM_SCAN_NEXT);
|
||||
if (TABLES_ON_MASTER) {
|
||||
assertCounter("ScanSize_num_ops", numScanNext);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
@ -58,7 +58,6 @@ import org.apache.hadoop.hbase.coprocessor.RegionObserver;
|
||||
import org.apache.hadoop.hbase.filter.BinaryComparator;
|
||||
import org.apache.hadoop.hbase.filter.RowFilter;
|
||||
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
|
||||
import org.apache.hadoop.hbase.master.LoadBalancer;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.junit.AfterClass;
|
||||
@ -109,9 +108,6 @@ public class TestRegionServerReadRequestMetrics {
|
||||
|
||||
@BeforeClass
|
||||
public static void setUpOnce() throws Exception {
|
||||
// Default starts one regionserver only.
|
||||
TEST_UTIL.getConfiguration().setBoolean(LoadBalancer.TABLES_ON_MASTER, true);
|
||||
// TEST_UTIL.getConfiguration().setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, true);
|
||||
TEST_UTIL.startMiniCluster();
|
||||
admin = TEST_UTIL.getAdmin();
|
||||
serverNames = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
|
||||
@ -146,16 +142,6 @@ public class TestRegionServerReadRequestMetrics {
|
||||
|
||||
assertEquals(expectedReadRequests,
|
||||
requestsMap.get(Metric.REGION_READ) - requestsMapPrev.get(Metric.REGION_READ));
|
||||
boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
|
||||
if (tablesOnMaster) {
|
||||
// If NO tables on master, then the single regionserver in this test carries user-space
|
||||
// tables and the meta table. The first time through, the read will be inflated by meta
|
||||
// lookups. We don't know which test will be first through since junit randomizes. This
|
||||
// method is used by a bunch of tests. Just do this check if master is hosting (system)
|
||||
// regions only.
|
||||
assertEquals(expectedReadRequests,
|
||||
requestsMap.get(Metric.SERVER_READ) - requestsMapPrev.get(Metric.SERVER_READ));
|
||||
}
|
||||
assertEquals(expectedFilteredReadRequests,
|
||||
requestsMap.get(Metric.FILTERED_REGION_READ)
|
||||
- requestsMapPrev.get(Metric.FILTERED_REGION_READ));
|
||||
|
||||
@ -34,7 +34,6 @@ import org.apache.hadoop.hbase.MiniHBaseCluster.MiniHBaseClusterRegionServer;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
|
||||
import org.apache.hadoop.hbase.master.HMaster;
|
||||
import org.apache.hadoop.hbase.master.LoadBalancer;
|
||||
import org.apache.hadoop.hbase.master.ServerManager;
|
||||
import org.apache.hadoop.hbase.testclassification.LargeTests;
|
||||
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
|
||||
@ -168,10 +167,8 @@ public class TestRegionServerReportForDuty {
|
||||
// Start a master and wait for it to become the active/primary master.
|
||||
// Use a random unique port
|
||||
cluster.getConfiguration().setInt(HConstants.MASTER_PORT, HBaseTestingUtility.randomFreePort());
|
||||
// master has a rs. defaultMinToStart = 2
|
||||
boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(testUtil.getConfiguration());
|
||||
cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, tablesOnMaster? 2: 1);
|
||||
cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, tablesOnMaster? 2: 1);
|
||||
cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
|
||||
cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 1);
|
||||
master = cluster.addMaster();
|
||||
rs = cluster.addRegionServer();
|
||||
LOG.debug("Starting master: " + master.getMaster().getServerName());
|
||||
@ -198,10 +195,8 @@ public class TestRegionServerReportForDuty {
|
||||
// Also let it wait for exactly 2 region severs to report in.
|
||||
// TODO: Add handling bindexception. Random port is not enough!!! Flakie test!
|
||||
cluster.getConfiguration().setInt(HConstants.MASTER_PORT, HBaseTestingUtility.randomFreePort());
|
||||
cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
|
||||
tablesOnMaster? 3: 2);
|
||||
cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART,
|
||||
tablesOnMaster? 3: 2);
|
||||
cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 2);
|
||||
cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2);
|
||||
backupMaster = cluster.addMaster();
|
||||
LOG.debug("Starting new master: " + backupMaster.getMaster().getServerName());
|
||||
backupMaster.start();
|
||||
@ -211,8 +206,7 @@ public class TestRegionServerReportForDuty {
|
||||
// Do some checking/asserts here.
|
||||
assertTrue(backupMaster.getMaster().isActiveMaster());
|
||||
assertTrue(backupMaster.getMaster().isInitialized());
|
||||
assertEquals(backupMaster.getMaster().getServerManager().getOnlineServersList().size(),
|
||||
tablesOnMaster? 3: 2);
|
||||
assertEquals(backupMaster.getMaster().getServerManager().getOnlineServersList().size(), 2);
|
||||
|
||||
}
|
||||
|
||||
@ -230,12 +224,8 @@ public class TestRegionServerReportForDuty {
|
||||
cluster.getConfiguration().setInt(HConstants.MASTER_PORT, HBaseTestingUtility.randomFreePort());
|
||||
// Override the default RS RPC retry interval of 100ms to 300ms
|
||||
cluster.getConfiguration().setLong("hbase.regionserver.rpc.retry.interval", 300);
|
||||
// master has a rs. defaultMinToStart = 2
|
||||
boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(testUtil.getConfiguration());
|
||||
cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
|
||||
tablesOnMaster ? 2 : 1);
|
||||
cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART,
|
||||
tablesOnMaster ? 2 : 1);
|
||||
cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
|
||||
cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 1);
|
||||
master = cluster.addMaster();
|
||||
rs = cluster.addRegionServer();
|
||||
LOG.debug("Starting master: " + master.getMaster().getServerName());
|
||||
@ -263,12 +253,8 @@ public class TestRegionServerReportForDuty {
|
||||
cluster.getConfiguration().setInt("hbase.procedure.remote.dispatcher.delay.msec", 0);
|
||||
cluster.getConfiguration().setLong("hbase.regionserver.rpc.retry.interval", 0);
|
||||
|
||||
// master has a rs. defaultMinToStart = 2
|
||||
boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(testUtil.getConfiguration());
|
||||
cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
|
||||
tablesOnMaster ? 2 : 1);
|
||||
cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART,
|
||||
tablesOnMaster ? 2 : 1);
|
||||
cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1);
|
||||
cluster.getConfiguration().setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 1);
|
||||
|
||||
// Inject manual environment edge for clock skew computation between RS and master
|
||||
ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
|
||||
|
||||
@ -76,7 +76,6 @@ import org.apache.hadoop.hbase.coprocessor.MasterObserver;
|
||||
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
|
||||
import org.apache.hadoop.hbase.io.Reference;
|
||||
import org.apache.hadoop.hbase.master.HMaster;
|
||||
import org.apache.hadoop.hbase.master.LoadBalancer;
|
||||
import org.apache.hadoop.hbase.master.MasterRpcServices;
|
||||
import org.apache.hadoop.hbase.master.RegionState;
|
||||
import org.apache.hadoop.hbase.master.RegionState.State;
|
||||
@ -923,14 +922,7 @@ public class TestSplitTransactionOnCluster {
|
||||
// hbase:meta We don't want hbase:meta replay polluting our test when we later crash
|
||||
// the table region serving server.
|
||||
int metaServerIndex = cluster.getServerWithMeta();
|
||||
boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TESTING_UTIL.getConfiguration());
|
||||
if (tablesOnMaster) {
|
||||
// Need to check master is supposed to host meta... perhaps it is not.
|
||||
throw new UnsupportedOperationException();
|
||||
// TODO: assertTrue(metaServerIndex == -1); // meta is on master now
|
||||
}
|
||||
HRegionServer metaRegionServer = tablesOnMaster?
|
||||
cluster.getMaster(): cluster.getRegionServer(metaServerIndex);
|
||||
HRegionServer metaRegionServer = cluster.getRegionServer(metaServerIndex);
|
||||
int tableRegionIndex = cluster.getServerWith(hri.getRegionName());
|
||||
assertTrue(tableRegionIndex != -1);
|
||||
HRegionServer tableRegionServer = cluster.getRegionServer(tableRegionIndex);
|
||||
|
||||
@ -35,7 +35,6 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
|
||||
import org.apache.hadoop.hbase.client.Put;
|
||||
import org.apache.hadoop.hbase.client.Table;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.master.LoadBalancer;
|
||||
import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
|
||||
import org.apache.hadoop.hbase.regionserver.HRegion;
|
||||
import org.apache.hadoop.hbase.regionserver.HRegionServer;
|
||||
@ -190,12 +189,6 @@ public class TestFlushWithThroughputController {
|
||||
// assertion here.
|
||||
assertTrue(regionServer.getFlushPressure() < pressure);
|
||||
Thread.sleep(5000);
|
||||
boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(hbtu.getConfiguration());
|
||||
if (tablesOnMaster) {
|
||||
// If no tables on the master, this math is off and I'm not sure what it is supposed to be
|
||||
// when meta is on the regionserver and not on the master.
|
||||
assertEquals(10L * 1024 * 1024, throughputController.getMaxThroughput(), EPSILON);
|
||||
}
|
||||
Table table = conn.getTable(tableName);
|
||||
Random rand = new Random();
|
||||
for (int i = 0; i < 10; i++) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user