From bc9184ee001c848518440c7e4dfa6e258a82089a Mon Sep 17 00:00:00 2001 From: niuyulin Date: Wed, 22 Apr 2020 14:55:38 +0800 Subject: [PATCH] HBASE-23933 Separate a hbase-balancer module (#1436) Signed-off-by: Duo Zhang Signed-off-by: Viraj Jasani --- hbase-assembly/pom.xml | 4 + .../src/main/assembly/hadoop-three-compat.xml | 1 + .../src/main/assembly/hadoop-two-compat.xml | 1 + hbase-balancer/pom.xml | 168 ++++++++++++++++++ .../favored/FavoredNodeAssignmentHelper.java | 9 +- .../hbase/favored/FavoredNodesPlan.java | 2 +- .../hbase/favored/FavoredNodesPromoter.java | 0 .../favored/StartcodeAgnosticServerName.java | 0 .../master/AssignmentVerificationReport.java | 2 +- .../hadoop/hbase/master/RackManager.java | 0 .../hadoop/hbase/master/RegionPlan.java | 0 .../SnapshotOfRegionAssignmentFromMeta.java | 0 .../hbase/master/assignment/ServerState.java | 0 .../master/balancer/BalancerRegionLoad.java | 0 .../master/balancer/ClusterLoadState.java | 0 .../master/balancer/MetricsBalancer.java | 6 +- .../balancer/MetricsStochasticBalancer.java | 0 .../master/balancer/RegionInfoComparator.java | 0 .../hbase/master/balancer/ServerAndLoad.java | 0 .../hadoop/hbase/master/TestRegionPlan.java | 0 .../master/balancer/TestServerAndLoad.java | 0 hbase-server/pom.xml | 4 + pom.xml | 6 + 23 files changed, 193 insertions(+), 10 deletions(-) create mode 100644 hbase-balancer/pom.xml rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodeAssignmentHelper.java (98%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesPlan.java (98%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesPromoter.java (100%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/favored/StartcodeAgnosticServerName.java (100%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java (99%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/RackManager.java (100%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/RegionPlan.java (100%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java (100%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/assignment/ServerState.java (100%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.java (100%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterLoadState.java (100%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancer.java (95%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsStochasticBalancer.java (100%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionInfoComparator.java (100%) rename {hbase-server => hbase-balancer}/src/main/java/org/apache/hadoop/hbase/master/balancer/ServerAndLoad.java (100%) rename {hbase-server => hbase-balancer}/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlan.java (100%) rename {hbase-server => hbase-balancer}/src/test/java/org/apache/hadoop/hbase/master/balancer/TestServerAndLoad.java (100%) diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml index 209ec702b03..198c9587701 100644 --- a/hbase-assembly/pom.xml +++ b/hbase-assembly/pom.xml @@ -289,6 +289,10 @@ org.apache.hbase hbase-backup + + org.apache.hbase + hbase-balancer + org.apache.hbase hbase-replication diff --git a/hbase-assembly/src/main/assembly/hadoop-three-compat.xml b/hbase-assembly/src/main/assembly/hadoop-three-compat.xml index 310d06c4a3e..b9c2777f7dd 100644 --- a/hbase-assembly/src/main/assembly/hadoop-three-compat.xml +++ b/hbase-assembly/src/main/assembly/hadoop-three-compat.xml @@ -50,6 +50,7 @@ org.apache.hbase:hbase-procedure org.apache.hbase:hbase-protocol org.apache.hbase:hbase-protocol-shaded + org.apache.hbase:hbase-balancer org.apache.hbase:hbase-replication org.apache.hbase:hbase-rest org.apache.hbase:hbase-rsgroup diff --git a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml index a769b65a64e..26b59ffb703 100644 --- a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml +++ b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml @@ -50,6 +50,7 @@ org.apache.hbase:hbase-procedure org.apache.hbase:hbase-protocol org.apache.hbase:hbase-protocol-shaded + org.apache.hbase:hbase-balancer org.apache.hbase:hbase-replication org.apache.hbase:hbase-rest org.apache.hbase:hbase-server diff --git a/hbase-balancer/pom.xml b/hbase-balancer/pom.xml new file mode 100644 index 00000000000..fba72eb4dd0 --- /dev/null +++ b/hbase-balancer/pom.xml @@ -0,0 +1,168 @@ + + + + 4.0.0 + + hbase-build-configuration + org.apache.hbase + 3.0.0-SNAPSHOT + ../hbase-build-configuration + + + hbase-balancer + Apache HBase - Balancer + HBase Balancer Support + + + + + + + org.apache.maven.plugins + maven-source-plugin + + + + maven-assembly-plugin + + true + + + + net.revelc.code + warbucks-maven-plugin + + + org.apache.maven.plugins + maven-checkstyle-plugin + + true + + + + + + + + org.apache.hbase + hbase-common + test-jar + test + + + org.apache.hbase + hbase-annotations + test-jar + test + + + org.apache.hbase + hbase-common + + + org.apache.hbase + hbase-client + + + org.slf4j + slf4j-api + + + com.github.stephenc.findbugs + findbugs-annotations + compile + true + + + junit + junit + test + + + + + + + + + hadoop-2.0 + + + + + !hadoop.profile + + + + + org.apache.hadoop + hadoop-common + + + + + + hadoop-3.0 + + + hadoop.profile + 3.0 + + + + + org.apache.hadoop + hadoop-common + + + + + eclipse-specific + + + m2e.version + + + + + + + + org.eclipse.m2e + lifecycle-mapping + + + + + + + + + + + + + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodeAssignmentHelper.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodeAssignmentHelper.java similarity index 98% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodeAssignmentHelper.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodeAssignmentHelper.java index a754c6b8e43..4b73522df6c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodeAssignmentHelper.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodeAssignmentHelper.java @@ -60,11 +60,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.FavoredNodes; /** - * Helper class for {@link FavoredNodeLoadBalancer} that has all the intelligence for racks, - * meta scans, etc. Instantiated by the {@link FavoredNodeLoadBalancer} when needed (from - * within calls like {@link FavoredNodeLoadBalancer#randomAssignment(RegionInfo, List)}). - * All updates to favored nodes should only be done from {@link FavoredNodesManager} and not - * through this helper class (except for tests). + * Helper class for FavoredNodeLoadBalancer that has all the intelligence for racks, meta scans, + * etc. Instantiated by the FavoredNodeLoadBalancer when needed (from within calls like + * FavoredNodeLoadBalancer#randomAssignment(RegionInfo, List). All updates to favored nodes should + * only be done from FavoredNodesManager and not through this helper class (except for tests). */ @InterfaceAudience.Private public class FavoredNodeAssignmentHelper { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesPlan.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesPlan.java similarity index 98% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesPlan.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesPlan.java index a7667c328a1..79207c01c12 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesPlan.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesPlan.java @@ -29,7 +29,7 @@ import org.apache.yetus.audience.InterfaceAudience; /** * This class contains the mapping information between each region name and - * its favored region server list. Used by {@link FavoredNodeLoadBalancer} set + * its favored region server list. Used by FavoredNodeLoadBalancer set * of classes and from unit tests (hence the class is public) * * All the access to this class is thread-safe. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesPromoter.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesPromoter.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesPromoter.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodesPromoter.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/favored/StartcodeAgnosticServerName.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/favored/StartcodeAgnosticServerName.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/favored/StartcodeAgnosticServerName.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/favored/StartcodeAgnosticServerName.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java similarity index 99% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java index 5c084bf4ebc..346555b1518 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/AssignmentVerificationReport.java @@ -35,7 +35,7 @@ import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Helper class that is used by {@link RegionPlacementMaintainer} to print + * Helper class that is used by RegionPlacementMaintainer to print * information for favored nodes * */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RackManager.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/RackManager.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/RackManager.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/RackManager.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlan.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/RegionPlan.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlan.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/RegionPlan.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/SnapshotOfRegionAssignmentFromMeta.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/ServerState.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/assignment/ServerState.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/ServerState.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/assignment/ServerState.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerRegionLoad.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterLoadState.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterLoadState.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterLoadState.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/ClusterLoadState.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancer.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancer.java similarity index 95% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancer.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancer.java index 1925568454b..015e1d486c1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancer.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsBalancer.java @@ -32,10 +32,10 @@ public class MetricsBalancer { public MetricsBalancer() { initSource(); } - + /** - * A function to instantiate the metrics source. This function can be overridden in its - * subclasses to provide extended sources + * A function to instantiate the metrics source. This function can be overridden in its subclasses + * to provide extended sources */ protected void initSource() { source = CompatibilitySingletonFactory.getInstance(MetricsBalancerSource.class); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsStochasticBalancer.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsStochasticBalancer.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsStochasticBalancer.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/MetricsStochasticBalancer.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionInfoComparator.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionInfoComparator.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionInfoComparator.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionInfoComparator.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/ServerAndLoad.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/ServerAndLoad.java similarity index 100% rename from hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/ServerAndLoad.java rename to hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/ServerAndLoad.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlan.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlan.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlan.java rename to hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlan.java diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestServerAndLoad.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestServerAndLoad.java similarity index 100% rename from hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestServerAndLoad.java rename to hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestServerAndLoad.java diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml index 48992b67bce..d947033b693 100644 --- a/hbase-server/pom.xml +++ b/hbase-server/pom.xml @@ -286,6 +286,10 @@ org.apache.hbase hbase-replication + + org.apache.hbase + hbase-balancer + org.apache.hbase hbase-common diff --git a/pom.xml b/pom.xml index e8c78c6e829..da4d92605e7 100755 --- a/pom.xml +++ b/pom.xml @@ -63,6 +63,7 @@ hbase-build-configuration hbase-replication + hbase-balancer hbase-mapreduce hbase-resource-bundle hbase-http @@ -1725,6 +1726,11 @@ org.apache.hbase ${project.version} + + hbase-balancer + org.apache.hbase + ${project.version} + hbase-http org.apache.hbase