From 623f8affe2c92208e1c63b3f7446a7fa6ce760b6 Mon Sep 17 00:00:00 2001 From: huaxiangsun Date: Mon, 23 May 2022 08:54:00 -0700 Subject: [PATCH] HBASE-26649 Support meta replica LoadBalance mode for RegionLocator#getAllRegionLocations() (#4442) Signed-off-by: Duo Zhang --- .../{client => }/CatalogReplicaMode.java | 4 +-- .../hadoop/hbase/ClientMetaTableAccessor.java | 34 ++++++++++++++++++- .../client/AsyncNonMetaRegionLocator.java | 1 + .../client/TestAsyncNonMetaRegionLocator.java | 1 + .../TestMetaRegionReplicaReplication.java | 21 ++++++++---- 5 files changed, 52 insertions(+), 9 deletions(-) rename hbase-client/src/main/java/org/apache/hadoop/hbase/{client => }/CatalogReplicaMode.java (96%) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/CatalogReplicaMode.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/CatalogReplicaMode.java similarity index 96% rename from hbase-client/src/main/java/org/apache/hadoop/hbase/client/CatalogReplicaMode.java rename to hbase-client/src/main/java/org/apache/hadoop/hbase/CatalogReplicaMode.java index 647d5dcf38f..b89673d45a8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/CatalogReplicaMode.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/CatalogReplicaMode.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hbase.client; +package org.apache.hadoop.hbase; import org.apache.yetus.audience.InterfaceAudience; @@ -33,7 +33,7 @@ import org.apache.yetus.audience.InterfaceAudience; * */ @InterfaceAudience.Private -enum CatalogReplicaMode { +public enum CatalogReplicaMode { NONE { @Override public String toString() { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClientMetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClientMetaTableAccessor.java index 14076f0d7b0..e948048325f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ClientMetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ClientMetaTableAccessor.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase; +import static org.apache.hadoop.hbase.client.RegionLocator.LOCATOR_META_REPLICAS_MODE; import static org.apache.hadoop.hbase.util.FutureUtils.addListener; import java.io.Closeable; @@ -26,6 +27,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; import org.apache.hadoop.hbase.client.AdvancedScanResultConsumer; import org.apache.hadoop.hbase.client.AsyncTable; @@ -257,7 +259,37 @@ public final class ClientMetaTableAccessor { } CompletableFuture future = new CompletableFuture(); - metaTable.scan(scan, new MetaTableScanResultConsumer(rowUpperLimit, visitor, future)); + // Get the region locator's meta replica mode. + CatalogReplicaMode metaReplicaMode = CatalogReplicaMode.fromString(metaTable.getConfiguration() + .get(LOCATOR_META_REPLICAS_MODE, CatalogReplicaMode.NONE.toString())); + + if (metaReplicaMode == CatalogReplicaMode.LOAD_BALANCE) { + addListener(metaTable.getDescriptor(), (desc, error) -> { + if (error != null) { + LOG.error("Failed to get meta table descriptor, error: ", error); + future.completeExceptionally(error); + return; + } + + int numOfReplicas = desc.getRegionReplication(); + if (numOfReplicas > 1) { + int replicaId = ThreadLocalRandom.current().nextInt(numOfReplicas); + + // When the replicaId is 0, do not set to Consistency.TIMELINE + if (replicaId > 0) { + scan.setReplicaId(replicaId); + scan.setConsistency(Consistency.TIMELINE); + } + } + metaTable.scan(scan, new MetaTableScanResultConsumer(rowUpperLimit, visitor, future)); + }); + } else { + if (metaReplicaMode == CatalogReplicaMode.HEDGED_READ) { + scan.setConsistency(Consistency.TIMELINE); + } + metaTable.scan(scan, new MetaTableScanResultConsumer(rowUpperLimit, visitor, future)); + } + return future; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java index 2e3b7f56691..2692a0b776b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncNonMetaRegionLocator.java @@ -50,6 +50,7 @@ import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.ObjectUtils; import org.apache.hadoop.hbase.CatalogFamilyFormat; +import org.apache.hadoop.hbase.CatalogReplicaMode; import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocator.java index 8a81d7ce774..f5f3cc4c6fb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocator.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncNonMetaRegionLocator.java @@ -37,6 +37,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadLocalRandom; import java.util.stream.IntStream; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CatalogReplicaMode; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HRegionLocation; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplication.java index 10399a8a062..411bdc53b04 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplication.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplication.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; @@ -396,11 +397,8 @@ public class TestMetaRegionReplicaReplication { } private void primaryIncreaseReplicaIncrease(final long[] before, final long[] after) { - // There are read requests increase for primary meta replica. - assertTrue(after[RegionInfo.DEFAULT_REPLICA_ID] > before[RegionInfo.DEFAULT_REPLICA_ID]); - - // There are read requests incrase for meta replica regions. - for (int i = 1; i < after.length; i++) { + // There are read requests increase for all meta replica regions, + for (int i = 0; i < after.length; i++) { assertTrue(after[i] > before[i]); } } @@ -420,6 +418,7 @@ public class TestMetaRegionReplicaReplication { final Region[] metaRegions = getAllRegions(TableName.META_TABLE_NAME, numOfMetaReplica); long[] readReqsForMetaReplicas = new long[numOfMetaReplica]; long[] readReqsForMetaReplicasAfterGet = new long[numOfMetaReplica]; + long[] readReqsForMetaReplicasAfterGetAllLocations = new long[numOfMetaReplica]; long[] readReqsForMetaReplicasAfterMove = new long[numOfMetaReplica]; long[] readReqsForMetaReplicasAfterSecondMove = new long[numOfMetaReplica]; long[] readReqsForMetaReplicasAfterThirdGet = new long[numOfMetaReplica]; @@ -468,6 +467,16 @@ public class TestMetaRegionReplicaReplication { // There are more reads against all meta replica regions, including the primary region. primaryIncreaseReplicaIncrease(readReqsForMetaReplicas, readReqsForMetaReplicasAfterGet); + RegionLocator locator = tableForGet.getRegionLocator(); + + for (int j = 0; j < numOfMetaReplica * 3; j ++) { + locator.getAllRegionLocations(); + } + + getMetaReplicaReadRequests(metaRegions, readReqsForMetaReplicasAfterGetAllLocations); + primaryIncreaseReplicaIncrease(readReqsForMetaReplicasAfterGet, + readReqsForMetaReplicasAfterGetAllLocations); + // move one of regions so it meta cache may be invalid. HTU.moveRegionAndWait(userRegion.getRegionInfo(), destRs.getServerName()); @@ -477,7 +486,7 @@ public class TestMetaRegionReplicaReplication { // There are read requests increase for primary meta replica. // For rest of meta replicas, there is no change as regionMove will tell the new location - primaryIncreaseReplicaNoChange(readReqsForMetaReplicasAfterGet, + primaryIncreaseReplicaNoChange(readReqsForMetaReplicasAfterGetAllLocations, readReqsForMetaReplicasAfterMove); // Move region again. HTU.moveRegionAndWait(userRegion.getRegionInfo(), srcRs.getServerName());