From 7218c83f81d2ecd243f36ca6a8d656be65eaf4ab Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Fri, 28 May 2021 15:57:59 +0800 Subject: [PATCH] HBASE-25931 Move FavoredNodeManager to hbase-balancer module (#3324) Signed-off-by: Yulin Niu Signed-off-by: Michael Stack --- hbase-balancer/pom.xml | 8 +++ .../hbase/favored/FavoredNodesManager.java | 54 +++++++++---------- .../master/balancer/ClusterInfoProvider.java | 6 +++ .../balancer/DummyClusterInfoProvider.java | 9 +++- .../balancer/FavoredStochasticBalancer.java | 3 +- .../balancer/MasterClusterInfoProvider.java | 7 ++- .../rsgroup/RSGroupBasedLoadBalancer.java | 2 +- 7 files changed, 55 insertions(+), 34 deletions(-) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesManager.java (88%) diff --git a/hbase-balancer/pom.xml b/hbase-balancer/pom.xml index 461d2ef9985..8ba6ab1a573 100644 --- a/hbase-balancer/pom.xml +++ b/hbase-balancer/pom.xml @@ -88,6 +88,14 @@ org.apache.hbase hbase-client + + org.apache.hadoop + hadoop-hdfs-client + + + org.apache.hadoop + hadoop-hdfs + org.slf4j slf4j-api diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesManager.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesManager.java similarity index 88% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesManager.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesManager.java index 16f0934a291..07ff8008a25 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesManager.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesManager.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position.PRIMARY; import static org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position.SECONDARY; import static org.apache.hadoop.hbase.favored.FavoredNodesPlan.Position.TERTIARY; +import com.google.errorprone.annotations.RestrictedApi; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -34,9 +35,8 @@ import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.RegionInfo; -import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.master.RackManager; import org.apache.hadoop.hbase.master.SnapshotOfRegionAssignmentFromMeta; +import org.apache.hadoop.hbase.master.balancer.ClusterInfoProvider; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.net.NetUtils; @@ -49,17 +49,18 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Maps; import org.apache.hbase.thirdparty.com.google.common.collect.Sets; /** - * FavoredNodesManager is responsible for maintaining favored nodes info in internal cache and - * META table. Its the centralized store for all favored nodes information. All reads and updates - * should be done through this class. There should only be one instance of - * {@link FavoredNodesManager} in Master. {@link FavoredNodesPlan} and favored node information - * from {@link SnapshotOfRegionAssignmentFromMeta} should not be used outside this class (except - * for tools that only read or fortest cases). All other classes including Favored balancers - * and {@link FavoredNodeAssignmentHelper} should use {@link FavoredNodesManager} for any + * FavoredNodesManager is responsible for maintaining favored nodes info in internal cache and META + * table. Its the centralized store for all favored nodes information. All reads and updates should + * be done through this class. There should only be one instance of {@link FavoredNodesManager} in + * Master. {@link FavoredNodesPlan} and favored node information from + * {@link SnapshotOfRegionAssignmentFromMeta} should not be used outside this class (except for + * tools that only read or fortest cases). All other classes including Favored balancers and + * {@link FavoredNodeAssignmentHelper} should use {@link FavoredNodesManager} for any * read/write/deletes to favored nodes. */ @InterfaceAudience.Private public class FavoredNodesManager { + private static final Logger LOG = LoggerFactory.getLogger(FavoredNodesManager.class); private final FavoredNodesPlan globalFavoredNodesAssignmentPlan; @@ -67,21 +68,19 @@ public class FavoredNodesManager { private final Map> secondaryRSToRegionMap; private final Map> teritiaryRSToRegionMap; - private final MasterServices masterServices; - private final RackManager rackManager; + private final ClusterInfoProvider provider; /** * Datanode port to be used for Favored Nodes. */ private int datanodeDataTransferPort; - public FavoredNodesManager(MasterServices masterServices) { - this.masterServices = masterServices; + public FavoredNodesManager(ClusterInfoProvider provider) { + this.provider = provider; this.globalFavoredNodesAssignmentPlan = new FavoredNodesPlan(); this.primaryRSToRegionMap = new HashMap<>(); this.secondaryRSToRegionMap = new HashMap<>(); this.teritiaryRSToRegionMap = new HashMap<>(); - this.rackManager = new RackManager(masterServices.getConfiguration()); } public synchronized void initialize(SnapshotOfRegionAssignmentFromMeta snapshot) { @@ -96,10 +95,12 @@ public class FavoredNodesManager { datanodeDataTransferPort= getDataNodePort(); } + @RestrictedApi(explanation = "Should only be called in tests", link = "", + allowedOnPath = ".*(/src/test/.*|FavoredNodesManager).java") public int getDataNodePort() { HdfsConfiguration.init(); - Configuration dnConf = new HdfsConfiguration(masterServices.getConfiguration()); + Configuration dnConf = new HdfsConfiguration(provider.getConfiguration()); int dnPort = NetUtils.createSocketAddr( dnConf.get(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, @@ -112,7 +113,7 @@ public class FavoredNodesManager { return this.globalFavoredNodesAssignmentPlan.getFavoredNodes(regionInfo); } - /* + /** * Favored nodes are not applicable for system tables. We will use this to check before * we apply any favored nodes logic on a region. */ @@ -128,7 +129,7 @@ public class FavoredNodesManager { return regions.stream().filter(r -> !isFavoredNodeApplicable(r)).collect(Collectors.toSet()); } - /* + /** * This should only be used when sending FN information to the region servers. Instead of * sending the region server port, we use the datanode port. This helps in centralizing the DN * port logic in Master. The RS uses the port from the favored node list as hints. @@ -148,7 +149,6 @@ public class FavoredNodesManager { public synchronized void updateFavoredNodes(Map> regionFNMap) throws IOException { - Map> regionToFavoredNodes = new HashMap<>(); for (Map.Entry> entry : regionFNMap.entrySet()) { RegionInfo regionInfo = entry.getKey(); @@ -187,7 +187,7 @@ public class FavoredNodesManager { // Lets do a bulk update to meta since that reduces the RPC's FavoredNodeAssignmentHelper.updateMetaWithFavoredNodesInfo(regionToFavoredNodes, - masterServices.getConnection()); + provider.getConnection()); deleteFavoredNodesForRegions(regionToFavoredNodes.keySet()); for (Map.Entry> entry : regionToFavoredNodes.entrySet()) { @@ -227,13 +227,13 @@ public class FavoredNodesManager { teritiaryRSToRegionMap.put(serverToUse, regionList); } - /* + /** * Get the replica count for the servers provided. - * - * For each server, replica count includes three counts for primary, secondary and tertiary. - * If a server is the primary favored node for 10 regions, secondary for 5 and tertiary - * for 1, then the list would be [10, 5, 1]. If the server is newly added to the cluster is - * not a favored node for any region, the replica count would be [0, 0, 0]. + *

+ * For each server, replica count includes three counts for primary, secondary and tertiary. If a + * server is the primary favored node for 10 regions, secondary for 5 and tertiary for 1, then the + * list would be [10, 5, 1]. If the server is newly added to the cluster is not a favored node for + * any region, the replica count would be [0, 0, 0]. */ public synchronized Map> getReplicaLoad(List servers) { Map> result = Maps.newHashMap(); @@ -297,8 +297,4 @@ public class FavoredNodesManager { } return regionInfos; } - - public RackManager getRackManager() { - return rackManager; - } } diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterInfoProvider.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterInfoProvider.java index 0721b7a58ae..a380eb791a4 100644 --- a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterInfoProvider.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterInfoProvider.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.BalancerDecision; import org.apache.hadoop.hbase.client.BalancerRejection; +import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.conf.ConfigurationObserver; @@ -47,6 +48,11 @@ public interface ClusterInfoProvider extends ConfigurationObserver { */ Configuration getConfiguration(); + /** + * Returns a reference to the cluster's connection. + */ + Connection getConnection(); + /** * Get all the regions of this cluster. *

diff --git a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/DummyClusterInfoProvider.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/DummyClusterInfoProvider.java index 3929e6d6716..5ac370efbdb 100644 --- a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/DummyClusterInfoProvider.java +++ b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/DummyClusterInfoProvider.java @@ -31,12 +31,13 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.BalancerDecision; import org.apache.hadoop.hbase.client.BalancerRejection; +import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; public class DummyClusterInfoProvider implements ClusterInfoProvider { - private final Configuration conf; + private volatile Configuration conf; public DummyClusterInfoProvider(Configuration conf) { this.conf = conf; @@ -47,6 +48,11 @@ public class DummyClusterInfoProvider implements ClusterInfoProvider { return conf; } + @Override + public Connection getConnection() { + return null; + } + @Override public List getAssignedRegions() { return Collections.emptyList(); @@ -99,5 +105,6 @@ public class DummyClusterInfoProvider implements ClusterInfoProvider { @Override public void onConfigurationChange(Configuration conf) { + this.conf = conf; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java index 72e65f24d92..f46478997e0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/FavoredStochasticBalancer.java @@ -118,8 +118,7 @@ public class FavoredStochasticBalancer extends StochasticLoadBalancer implements Set regionSet = new HashSet<>(regions); Map> assignmentMap = new HashMap<>(); try { - FavoredNodeAssignmentHelper helper = - new FavoredNodeAssignmentHelper(servers, fnm.getRackManager()); + FavoredNodeAssignmentHelper helper = new FavoredNodeAssignmentHelper(servers, rackManager); helper.initialize(); Set systemRegions = FavoredNodesManager.filterNonFNApplicableRegions(regionSet); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MasterClusterInfoProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MasterClusterInfoProvider.java index ea694cc47ba..c061b448674 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MasterClusterInfoProvider.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MasterClusterInfoProvider.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.BalancerDecision; import org.apache.hadoop.hbase.client.BalancerRejection; +import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.master.MasterServices; @@ -88,6 +89,11 @@ public class MasterClusterInfoProvider implements ClusterInfoProvider { return services.getConfiguration(); } + @Override + public Connection getConnection() { + return services.getConnection(); + } + @Override public List getAssignedRegions() { AssignmentManager am = services.getAssignmentManager(); @@ -169,5 +175,4 @@ public class MasterClusterInfoProvider implements ClusterInfoProvider { NamedQueueRecorder getNamedQueueRecorder() { return namedQueueRecorder; } - } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java index a4730c8d736..5f8552aa938 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java @@ -355,7 +355,7 @@ public class RSGroupBasedLoadBalancer implements LoadBalancer { internalBalancer.setClusterInfoProvider(provider); // special handling for favor node balancers if (internalBalancer instanceof FavoredNodesPromoter) { - favoredNodesManager = new FavoredNodesManager(masterServices); + favoredNodesManager = new FavoredNodesManager(provider); if (internalBalancer instanceof FavoredNodeLoadBalancer) { ((FavoredNodeLoadBalancer) internalBalancer).setMasterServices(masterServices); }