HBASE-24416 RegionNormalizer spliting region should not be limited by hbase.normalizer.min.region.count (#1770)
Signed-off-by: Guanghao Zhang <zghao@apache.org>
This commit is contained in:
parent
29c59c0ba8
commit
e06e1dba10
|
@ -22,6 +22,7 @@ import java.io.IOException;
|
|||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
import org.apache.hadoop.hbase.RegionMetrics;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.Size;
|
||||
|
@ -42,8 +43,14 @@ import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
|
|||
@InterfaceAudience.Private
|
||||
public abstract class AbstractRegionNormalizer implements RegionNormalizer {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(AbstractRegionNormalizer.class);
|
||||
|
||||
public static final String HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY =
|
||||
"hbase.normalizer.min.region.count";
|
||||
public static final int HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT = 3;
|
||||
|
||||
protected MasterServices masterServices;
|
||||
protected MasterRpcServices masterRpcServices;
|
||||
protected int minRegionCount;
|
||||
|
||||
/**
|
||||
* Set the master service.
|
||||
|
@ -52,6 +59,9 @@ public abstract class AbstractRegionNormalizer implements RegionNormalizer {
|
|||
@Override
|
||||
public void setMasterServices(MasterServices masterServices) {
|
||||
this.masterServices = masterServices;
|
||||
minRegionCount = masterServices.getConfiguration().getInt(
|
||||
HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY,
|
||||
HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_DEFAULT);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -23,10 +23,10 @@ import java.util.ArrayList;
|
|||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
import org.apache.hadoop.hbase.HBaseIOException;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.client.RegionInfo;
|
||||
import org.apache.hadoop.hbase.master.MasterServices;
|
||||
import org.apache.yetus.audience.InterfaceAudience;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
@ -63,13 +63,13 @@ import org.slf4j.LoggerFactory;
|
|||
public class MergeNormalizer extends AbstractRegionNormalizer {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(MergeNormalizer.class);
|
||||
|
||||
private int minRegionCount;
|
||||
private int minRegionAge;
|
||||
private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length];
|
||||
|
||||
public MergeNormalizer() {
|
||||
Configuration conf = HBaseConfiguration.create();
|
||||
minRegionCount = conf.getInt("hbase.normalizer.min.region.count", 3);
|
||||
@Override
|
||||
public void setMasterServices(MasterServices masterServices) {
|
||||
super.setMasterServices(masterServices);
|
||||
Configuration conf = masterServices.getConfiguration();
|
||||
minRegionAge = conf.getInt("hbase.normalizer.min.region.merge.age", 3);
|
||||
}
|
||||
|
||||
|
|
|
@ -22,7 +22,7 @@ import java.util.ArrayList;
|
|||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
|
||||
import org.apache.hadoop.hbase.HBaseIOException;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.client.RegionInfo;
|
||||
|
@ -51,13 +51,8 @@ import org.slf4j.LoggerFactory;
|
|||
public class SimpleRegionNormalizer extends AbstractRegionNormalizer {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(SimpleRegionNormalizer.class);
|
||||
private int minRegionCount;
|
||||
private static long[] skippedCount = new long[NormalizationPlan.PlanType.values().length];
|
||||
|
||||
public SimpleRegionNormalizer() {
|
||||
minRegionCount = HBaseConfiguration.create().getInt("hbase.normalizer.min.region.count", 3);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void planSkipped(RegionInfo hri, PlanType type) {
|
||||
skippedCount[type.ordinal()]++;
|
||||
|
@ -112,11 +107,7 @@ public class SimpleRegionNormalizer extends AbstractRegionNormalizer {
|
|||
List<RegionInfo> tableRegions =
|
||||
masterServices.getAssignmentManager().getRegionStates().getRegionsOfTable(table);
|
||||
|
||||
if (tableRegions == null || tableRegions.size() < minRegionCount) {
|
||||
int nrRegions = tableRegions == null ? 0 : tableRegions.size();
|
||||
LOG.debug("Table {} has {} regions, required min number of regions for normalizer to run is "
|
||||
+ "{}, not running normalizer",
|
||||
table, nrRegions, minRegionCount);
|
||||
if (tableRegions == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
@ -131,9 +122,15 @@ public class SimpleRegionNormalizer extends AbstractRegionNormalizer {
|
|||
}
|
||||
|
||||
if (mergeEnabled) {
|
||||
List<NormalizationPlan> mergePlans = getMergeNormalizationPlan(table);
|
||||
if (mergePlans != null) {
|
||||
plans.addAll(mergePlans);
|
||||
if (tableRegions.size() < minRegionCount) {
|
||||
LOG.debug("Table {} has {} regions, required min number of regions for normalizer to run" +
|
||||
" is {}, not running normalizer",
|
||||
table, tableRegions.size(), minRegionCount);
|
||||
} else {
|
||||
List<NormalizationPlan> mergePlans = getMergeNormalizationPlan(table);
|
||||
if (mergePlans != null) {
|
||||
plans.addAll(mergePlans);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (plans.isEmpty()) {
|
||||
|
|
|
@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.master.normalizer;
|
|||
|
||||
import static java.lang.String.format;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
|
@ -30,7 +31,10 @@ import java.util.HashMap;
|
|||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
import org.apache.hadoop.hbase.HBaseIOException;
|
||||
import org.apache.hadoop.hbase.RegionMetrics;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
|
@ -526,6 +530,45 @@ public class TestSimpleRegionNormalizer {
|
|||
assertEquals(hri2, ((MergeNormalizationPlan) plan).getSecondRegion());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSplitIfTooFewRegions() throws HBaseIOException {
|
||||
final TableName tableName = TableName.valueOf(name.getMethodName());
|
||||
List<RegionInfo> RegionInfo = new ArrayList<>();
|
||||
Map<byte[], Integer> regionSizes = new HashMap<>();
|
||||
|
||||
RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName)
|
||||
.setStartKey(Bytes.toBytes("aaa"))
|
||||
.setEndKey(Bytes.toBytes("bbb"))
|
||||
.build();
|
||||
RegionInfo.add(hri1);
|
||||
regionSizes.put(hri1.getRegionName(), 1);
|
||||
|
||||
RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName)
|
||||
.setStartKey(Bytes.toBytes("bbb"))
|
||||
.setEndKey(Bytes.toBytes("ccc"))
|
||||
.build();
|
||||
RegionInfo.add(hri2);
|
||||
regionSizes.put(hri2.getRegionName(), 1);
|
||||
// the third region is huge one
|
||||
RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName)
|
||||
.setStartKey(Bytes.toBytes("ccc"))
|
||||
.setEndKey(Bytes.toBytes("ddd"))
|
||||
.build();
|
||||
RegionInfo.add(hri3);
|
||||
regionSizes.put(hri3.getRegionName(), 10);
|
||||
|
||||
setupMocksForNormalizer(regionSizes, RegionInfo);
|
||||
|
||||
Configuration configuration = HBaseConfiguration.create();
|
||||
configuration.setInt(AbstractRegionNormalizer.HBASE_REGION_NORMALIZER_MIN_REGION_COUNT_KEY, 4);
|
||||
when(masterServices.getConfiguration()).thenReturn(configuration);
|
||||
|
||||
List<NormalizationPlan> plans = normalizer.computePlanForTable(tableName);
|
||||
assertNotNull(plans);
|
||||
NormalizationPlan plan = plans.get(0);
|
||||
assertEquals(hri3, ((SplitNormalizationPlan) plan).getRegionInfo());
|
||||
}
|
||||
|
||||
@SuppressWarnings("MockitoCast")
|
||||
private void setupMocksForNormalizer(Map<byte[], Integer> regionSizes,
|
||||
List<RegionInfo> RegionInfo) {
|
||||
|
|
Loading…
Reference in New Issue