HBASE-24416 RegionNormalizer spliting region should not be limited by hbase.normalizer.min.region.count (#1770)

Signed-off-by: Guanghao Zhang <zghao@apache.org>
This commit is contained in:
XinSun 2020-05-26 17:03:14 +08:00 committed by GitHub
parent 29c59c0ba8
commit e06e1dba10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 69 additions and 19 deletions

View File

@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import org.apache.hadoop.hbase.RegionMetrics;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.Size;
@ -42,8 +43,14 @@ import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
@InterfaceAudience.Private
public abstract class AbstractRegionNormalizer implements RegionNormalizer {
private static final Logger LOG = LoggerFactory.getLogger(AbstractRegionNormalizer.class);
public static final String HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY =
"hbase.normalizer.min.region.count";
public static final int HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT = 3;
protected MasterServices masterServices;
protected MasterRpcServices masterRpcServices;
protected int minRegionCount;
/**
* Set the master service.
@ -52,6 +59,9 @@ public abstract class AbstractRegionNormalizer implements RegionNormalizer {
@Override
public void setMasterServices(MasterServices masterServices) {
this.masterServices = masterServices;
minRegionCount = masterServices.getConfiguration().getInt(
HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY,
HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT);
}
@Override

View File

@ -23,10 +23,10 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -63,13 +63,13 @@ import org.slf4j.LoggerFactory;
public class MergeNormalizer extends AbstractRegionNormalizer {
private static final Logger LOG = LoggerFactory.getLogger(MergeNormalizer.class);
private int minRegionCount;
private int minRegionAge;
private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length];
public MergeNormalizer() {
Configuration conf = HBaseConfiguration.create();
minRegionCount = conf.getInt("hbase.normalizer.min.region.count", 3);
@Override
public void setMasterServices(MasterServices masterServices) {
super.setMasterServices(masterServices);
Configuration conf = masterServices.getConfiguration();
minRegionAge = conf.getInt("hbase.normalizer.min.region.merge.age", 3);
}

View File

@ -22,7 +22,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
@ -51,13 +51,8 @@ import org.slf4j.LoggerFactory;
public class SimpleRegionNormalizer extends AbstractRegionNormalizer {
private static final Logger LOG = LoggerFactory.getLogger(SimpleRegionNormalizer.class);
private int minRegionCount;
private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length];
public SimpleRegionNormalizer() {
minRegionCount = HBaseConfiguration.create().getInt("hbase.normalizer.min.region.count", 3);
}
@Override
public void planSkipped(RegionInfo hri, PlanType type) {
skippedCount[type.ordinal()]++;
@ -112,11 +107,7 @@ public class SimpleRegionNormalizer extends AbstractRegionNormalizer {
List<RegionInfo> tableRegions =
masterServices.getAssignmentManager().getRegionStates().getRegionsOfTable(table);
if (tableRegions == null || tableRegions.size() < minRegionCount) {
int nrRegions = tableRegions == null ? 0 : tableRegions.size();
LOG.debug("Table {} has {} regions, required min number of regions for normalizer to run is "
+ "{}, not running normalizer",
table, nrRegions, minRegionCount);
if (tableRegions == null) {
return null;
}
@ -131,9 +122,15 @@ public class SimpleRegionNormalizer extends AbstractRegionNormalizer {
}
if (mergeEnabled) {
List<NormalizationPlan> mergePlans = getMergeNormalizationPlan(table);
if (mergePlans != null) {
plans.addAll(mergePlans);
if (tableRegions.size() < minRegionCount) {
LOG.debug("Table {} has {} regions, required min number of regions for normalizer to run" +
" is {}, not running normalizer",
table, tableRegions.size(), minRegionCount);
} else {
List<NormalizationPlan> mergePlans = getMergeNormalizationPlan(table);
if (mergePlans != null) {
plans.addAll(mergePlans);
}
}
}
if (plans.isEmpty()) {

View File

@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.master.normalizer;
import static java.lang.String.format;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@ -30,7 +31,10 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.RegionMetrics;
import org.apache.hadoop.hbase.ServerName;
@ -526,6 +530,45 @@ public class TestSimpleRegionNormalizer {
assertEquals(hri2, ((MergeNormalizationPlan) plan).getSecondRegion());
}
@Test
public void testSplitIfTooFewRegions() throws HBaseIOException {
final TableName tableName = TableName.valueOf(name.getMethodName());
List<RegionInfo> RegionInfo = new ArrayList<>();
Map<byte[], Integer> regionSizes = new HashMap<>();
RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
.setStartKey(Bytes.toBytes("aaa"))
.setEndKey(Bytes.toBytes("bbb"))
.build();
RegionInfo.add(hri1);
regionSizes.put(hri1.getRegionName(), 1);
RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
.setStartKey(Bytes.toBytes("bbb"))
.setEndKey(Bytes.toBytes("ccc"))
.build();
RegionInfo.add(hri2);
regionSizes.put(hri2.getRegionName(), 1);
// the third region is huge one
RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName)
.setStartKey(Bytes.toBytes("ccc"))
.setEndKey(Bytes.toBytes("ddd"))
.build();
RegionInfo.add(hri3);
regionSizes.put(hri3.getRegionName(), 10);
setupMocksForNormalizer(regionSizes, RegionInfo);
Configuration configuration = HBaseConfiguration.create();
configuration.setInt(AbstractRegionNormalizer.HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY, 4);
when(masterServices.getConfiguration()).thenReturn(configuration);
List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
assertNotNull(plans);
NormalizationPlan plan = plans.get(0);
assertEquals(hri3, ((SplitNormalizationPlan) plan).getRegionInfo());
}
@SuppressWarnings("MockitoCast")
private void setupMocksForNormalizer(Map<byte[], Integer> regionSizes,
List<RegionInfo> RegionInfo) {