HBASE-26649 Support meta replica LoadBalance mode for RegionLocator#getAllRegionLocations() (#4442)

Signed-off-by: Duo Zhang <zhangduo@apache.org>
This commit is contained in:
huaxiangsun 2022-05-23 08:54:00 -07:00 committed by GitHub
parent 1be2c83b3d
commit 623f8affe2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 52 additions and 9 deletions

View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.hbase.client; package org.apache.hadoop.hbase;
import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceAudience;
@ -33,7 +33,7 @@ import org.apache.yetus.audience.InterfaceAudience;
* </ol> * </ol>
*/ */
@InterfaceAudience.Private @InterfaceAudience.Private
enum CatalogReplicaMode { public enum CatalogReplicaMode {
NONE { NONE {
@Override @Override
public String toString() { public String toString() {

View File

@ -17,6 +17,7 @@
*/ */
package org.apache.hadoop.hbase; package org.apache.hadoop.hbase;
import static org.apache.hadoop.hbase.client.RegionLocator.LOCATOR_META_REPLICAS_MODE;
import static org.apache.hadoop.hbase.util.FutureUtils.addListener; import static org.apache.hadoop.hbase.util.FutureUtils.addListener;
import java.io.Closeable; import java.io.Closeable;
@ -26,6 +27,7 @@ import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import org.apache.hadoop.hbase.client.AdvancedScanResultConsumer; import org.apache.hadoop.hbase.client.AdvancedScanResultConsumer;
import org.apache.hadoop.hbase.client.AsyncTable; import org.apache.hadoop.hbase.client.AsyncTable;
@ -257,7 +259,37 @@ public final class ClientMetaTableAccessor {
} }
CompletableFuture<Void> future = new CompletableFuture<Void>(); CompletableFuture<Void> future = new CompletableFuture<Void>();
metaTable.scan(scan, new MetaTableScanResultConsumer(rowUpperLimit, visitor, future)); // Get the region locator's meta replica mode.
CatalogReplicaMode metaReplicaMode = CatalogReplicaMode.fromString(metaTable.getConfiguration()
.get(LOCATOR_META_REPLICAS_MODE, CatalogReplicaMode.NONE.toString()));
if (metaReplicaMode == CatalogReplicaMode.LOAD_BALANCE) {
addListener(metaTable.getDescriptor(), (desc, error) -> {
if (error != null) {
LOG.error("Failed to get meta table descriptor, error: ", error);
future.completeExceptionally(error);
return;
}
int numOfReplicas = desc.getRegionReplication();
if (numOfReplicas > 1) {
int replicaId = ThreadLocalRandom.current().nextInt(numOfReplicas);
// When the replicaId is 0, do not set to Consistency.TIMELINE
if (replicaId > 0) {
scan.setReplicaId(replicaId);
scan.setConsistency(Consistency.TIMELINE);
}
}
metaTable.scan(scan, new MetaTableScanResultConsumer(rowUpperLimit, visitor, future));
});
} else {
if (metaReplicaMode == CatalogReplicaMode.HEDGED_READ) {
scan.setConsistency(Consistency.TIMELINE);
}
metaTable.scan(scan, new MetaTableScanResultConsumer(rowUpperLimit, visitor, future));
}
return future; return future;
} }

View File

@ -50,6 +50,7 @@ import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
import org.apache.hadoop.hbase.CatalogFamilyFormat; import org.apache.hadoop.hbase.CatalogFamilyFormat;
import org.apache.hadoop.hbase.CatalogReplicaMode;
import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HRegionLocation;

View File

@ -37,6 +37,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.IntStream; import java.util.stream.IntStream;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CatalogReplicaMode;
import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil; import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HRegionLocation;

View File

@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.Table;
@ -396,11 +397,8 @@ public class TestMetaRegionReplicaReplication {
} }
private void primaryIncreaseReplicaIncrease(final long[] before, final long[] after) { private void primaryIncreaseReplicaIncrease(final long[] before, final long[] after) {
// There are read requests increase for primary meta replica. // There are read requests increase for all meta replica regions,
assertTrue(after[RegionInfo.DEFAULT_REPLICA_ID] > before[RegionInfo.DEFAULT_REPLICA_ID]); for (int i = 0; i < after.length; i++) {
// There are read requests incrase for meta replica regions.
for (int i = 1; i < after.length; i++) {
assertTrue(after[i] > before[i]); assertTrue(after[i] > before[i]);
} }
} }
@ -420,6 +418,7 @@ public class TestMetaRegionReplicaReplication {
final Region[] metaRegions = getAllRegions(TableName.META_TABLE_NAME, numOfMetaReplica); final Region[] metaRegions = getAllRegions(TableName.META_TABLE_NAME, numOfMetaReplica);
long[] readReqsForMetaReplicas = new long[numOfMetaReplica]; long[] readReqsForMetaReplicas = new long[numOfMetaReplica];
long[] readReqsForMetaReplicasAfterGet = new long[numOfMetaReplica]; long[] readReqsForMetaReplicasAfterGet = new long[numOfMetaReplica];
long[] readReqsForMetaReplicasAfterGetAllLocations = new long[numOfMetaReplica];
long[] readReqsForMetaReplicasAfterMove = new long[numOfMetaReplica]; long[] readReqsForMetaReplicasAfterMove = new long[numOfMetaReplica];
long[] readReqsForMetaReplicasAfterSecondMove = new long[numOfMetaReplica]; long[] readReqsForMetaReplicasAfterSecondMove = new long[numOfMetaReplica];
long[] readReqsForMetaReplicasAfterThirdGet = new long[numOfMetaReplica]; long[] readReqsForMetaReplicasAfterThirdGet = new long[numOfMetaReplica];
@ -468,6 +467,16 @@ public class TestMetaRegionReplicaReplication {
// There are more reads against all meta replica regions, including the primary region. // There are more reads against all meta replica regions, including the primary region.
primaryIncreaseReplicaIncrease(readReqsForMetaReplicas, readReqsForMetaReplicasAfterGet); primaryIncreaseReplicaIncrease(readReqsForMetaReplicas, readReqsForMetaReplicasAfterGet);
RegionLocator locator = tableForGet.getRegionLocator();
for (int j = 0; j < numOfMetaReplica * 3; j ++) {
locator.getAllRegionLocations();
}
getMetaReplicaReadRequests(metaRegions, readReqsForMetaReplicasAfterGetAllLocations);
primaryIncreaseReplicaIncrease(readReqsForMetaReplicasAfterGet,
readReqsForMetaReplicasAfterGetAllLocations);
// move one of regions so it meta cache may be invalid. // move one of regions so it meta cache may be invalid.
HTU.moveRegionAndWait(userRegion.getRegionInfo(), destRs.getServerName()); HTU.moveRegionAndWait(userRegion.getRegionInfo(), destRs.getServerName());
@ -477,7 +486,7 @@ public class TestMetaRegionReplicaReplication {
// There are read requests increase for primary meta replica. // There are read requests increase for primary meta replica.
// For rest of meta replicas, there is no change as regionMove will tell the new location // For rest of meta replicas, there is no change as regionMove will tell the new location
primaryIncreaseReplicaNoChange(readReqsForMetaReplicasAfterGet, primaryIncreaseReplicaNoChange(readReqsForMetaReplicasAfterGetAllLocations,
readReqsForMetaReplicasAfterMove); readReqsForMetaReplicasAfterMove);
// Move region again. // Move region again.
HTU.moveRegionAndWait(userRegion.getRegionInfo(), srcRs.getServerName()); HTU.moveRegionAndWait(userRegion.getRegionInfo(), srcRs.getServerName());