HBASE-24648 Remove the legacy 'forceSplit' related code at region server side (#1990)
Signed-off-by: Viraj Jasani <vjasani@apache.org>
This commit is contained in:
parent
ed6104418c
commit
3effd28a75
|
@ -167,12 +167,15 @@ public class SplitTableRegionProcedure
|
||||||
return daughterTwoRI;
|
return daughterTwoRI;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private boolean hasBestSplitRow() {
|
||||||
|
return bestSplitRow != null && bestSplitRow.length > 0;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check whether the region is splittable
|
* Check whether the region is splittable
|
||||||
* @param env MasterProcedureEnv
|
* @param env MasterProcedureEnv
|
||||||
* @param regionToSplit parent Region to be split
|
* @param regionToSplit parent Region to be split
|
||||||
* @param splitRow if splitRow is not specified, will first try to get bestSplitRow from RS
|
* @param splitRow if splitRow is not specified, will first try to get bestSplitRow from RS
|
||||||
* @throws IOException
|
|
||||||
*/
|
*/
|
||||||
private void checkSplittable(final MasterProcedureEnv env,
|
private void checkSplittable(final MasterProcedureEnv env,
|
||||||
final RegionInfo regionToSplit, final byte[] splitRow) throws IOException {
|
final RegionInfo regionToSplit, final byte[] splitRow) throws IOException {
|
||||||
|
@ -187,19 +190,20 @@ public class SplitTableRegionProcedure
|
||||||
boolean splittable = false;
|
boolean splittable = false;
|
||||||
if (node != null) {
|
if (node != null) {
|
||||||
try {
|
try {
|
||||||
if (bestSplitRow == null || bestSplitRow.length == 0) {
|
GetRegionInfoResponse response;
|
||||||
LOG
|
if (!hasBestSplitRow()) {
|
||||||
.info("splitKey isn't explicitly specified, will try to find a best split key from RS");
|
LOG.info(
|
||||||
}
|
"{} splitKey isn't explicitly specified, will try to find a best split key from RS {}",
|
||||||
// Always set bestSplitRow request as true here,
|
node.getRegionInfo().getRegionNameAsString(), node.getRegionLocation());
|
||||||
// need to call Region#checkSplit to check it splittable or not
|
response = AssignmentManagerUtil.getRegionInfoResponse(env, node.getRegionLocation(),
|
||||||
GetRegionInfoResponse response = AssignmentManagerUtil.getRegionInfoResponse(env,
|
node.getRegionInfo(), true);
|
||||||
node.getRegionLocation(), node.getRegionInfo(), true);
|
bestSplitRow =
|
||||||
if(bestSplitRow == null || bestSplitRow.length == 0) {
|
response.hasBestSplitRow() ? response.getBestSplitRow().toByteArray() : null;
|
||||||
bestSplitRow = response.hasBestSplitRow() ? response.getBestSplitRow().toByteArray() : null;
|
} else {
|
||||||
|
response = AssignmentManagerUtil.getRegionInfoResponse(env, node.getRegionLocation(),
|
||||||
|
node.getRegionInfo(), false);
|
||||||
}
|
}
|
||||||
splittable = response.hasSplittable() && response.getSplittable();
|
splittable = response.hasSplittable() && response.getSplittable();
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
LOG.debug("Splittable=" + splittable + " " + node.toShortString());
|
LOG.debug("Splittable=" + splittable + " " + node.toShortString());
|
||||||
}
|
}
|
||||||
|
|
|
@ -191,7 +191,7 @@ public class CompactSplit implements CompactionRequester, PropagatingConfigurati
|
||||||
HRegion hr = (HRegion)r;
|
HRegion hr = (HRegion)r;
|
||||||
try {
|
try {
|
||||||
if (shouldSplitRegion() && hr.getCompactPriority() >= PRIORITY_USER) {
|
if (shouldSplitRegion() && hr.getCompactPriority() >= PRIORITY_USER) {
|
||||||
byte[] midKey = hr.checkSplit();
|
byte[] midKey = hr.checkSplit().orElse(null);
|
||||||
if (midKey != null) {
|
if (midKey != null) {
|
||||||
requestSplit(r, midKey);
|
requestSplit(r, midKey);
|
||||||
return true;
|
return true;
|
||||||
|
@ -216,9 +216,6 @@ public class CompactSplit implements CompactionRequester, PropagatingConfigurati
|
||||||
if (midKey == null) {
|
if (midKey == null) {
|
||||||
LOG.debug("Region " + r.getRegionInfo().getRegionNameAsString() +
|
LOG.debug("Region " + r.getRegionInfo().getRegionNameAsString() +
|
||||||
" not splittable because midkey=null");
|
" not splittable because midkey=null");
|
||||||
if (((HRegion)r).shouldForceSplit()) {
|
|
||||||
((HRegion)r).clearSplit();
|
|
||||||
}
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -68,7 +68,6 @@ public class ConstantSizeRegionSplitPolicy extends RegionSplitPolicy {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected boolean shouldSplit() {
|
protected boolean shouldSplit() {
|
||||||
boolean force = region.shouldForceSplit();
|
|
||||||
boolean foundABigStore = false;
|
boolean foundABigStore = false;
|
||||||
|
|
||||||
for (HStore store : region.getStores()) {
|
for (HStore store : region.getStores()) {
|
||||||
|
@ -84,7 +83,7 @@ public class ConstantSizeRegionSplitPolicy extends RegionSplitPolicy {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return foundABigStore || force;
|
return foundABigStore;
|
||||||
}
|
}
|
||||||
|
|
||||||
long getDesiredMaxFileSize() {
|
long getDesiredMaxFileSize() {
|
||||||
|
|
|
@ -22,15 +22,20 @@ import org.apache.yetus.audience.InterfaceAudience;
|
||||||
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
|
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A {@link RegionSplitPolicy} that disables region splits.
|
* A {@link RegionSplitPolicy} that disables region splits. This should be used with care, since it
|
||||||
* This should be used with care, since it will disable automatic sharding.
|
* will disable automatic sharding. Most of the time, using {@link ConstantSizeRegionSplitPolicy}
|
||||||
* Most of the time, using {@link ConstantSizeRegionSplitPolicy} with a
|
* with a large region size (10GB, etc) is safer.
|
||||||
* large region size (10GB, etc) is safer.
|
|
||||||
*/
|
*/
|
||||||
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
|
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
|
||||||
public class DisabledRegionSplitPolicy extends RegionSplitPolicy {
|
public class DisabledRegionSplitPolicy extends RegionSplitPolicy {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected boolean shouldSplit() {
|
protected boolean shouldSplit() {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected boolean canSplit() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -691,8 +691,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
|
||||||
|
|
||||||
// Stop updates lock
|
// Stop updates lock
|
||||||
private final ReentrantReadWriteLock updatesLock = new ReentrantReadWriteLock();
|
private final ReentrantReadWriteLock updatesLock = new ReentrantReadWriteLock();
|
||||||
private boolean splitRequest;
|
|
||||||
private byte[] explicitSplitPoint = null;
|
|
||||||
|
|
||||||
private final MultiVersionConcurrencyControl mvcc;
|
private final MultiVersionConcurrencyControl mvcc;
|
||||||
|
|
||||||
|
@ -1463,7 +1461,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean isSplittable() {
|
public boolean isSplittable() {
|
||||||
return isAvailable() && !hasReferences();
|
return splitPolicy.canSplit();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -1962,7 +1960,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
|
||||||
/**
|
/**
|
||||||
* @return split policy for this region.
|
* @return split policy for this region.
|
||||||
*/
|
*/
|
||||||
public RegionSplitPolicy getSplitPolicy() {
|
@VisibleForTesting
|
||||||
|
RegionSplitPolicy getSplitPolicy() {
|
||||||
return this.splitPolicy;
|
return this.splitPolicy;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -8346,9 +8345,9 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
|
||||||
|
|
||||||
public static final long FIXED_OVERHEAD = ClassSize.align(
|
public static final long FIXED_OVERHEAD = ClassSize.align(
|
||||||
ClassSize.OBJECT +
|
ClassSize.OBJECT +
|
||||||
ClassSize.ARRAY +
|
56 * ClassSize.REFERENCE +
|
||||||
55 * ClassSize.REFERENCE + 3 * Bytes.SIZEOF_INT +
|
3 * Bytes.SIZEOF_INT +
|
||||||
(15 * Bytes.SIZEOF_LONG) +
|
14 * Bytes.SIZEOF_LONG +
|
||||||
3 * Bytes.SIZEOF_BOOLEAN);
|
3 * Bytes.SIZEOF_BOOLEAN);
|
||||||
|
|
||||||
// woefully out of date - currently missing:
|
// woefully out of date - currently missing:
|
||||||
|
@ -8483,51 +8482,27 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
|
||||||
return responseBuilder.build();
|
return responseBuilder.build();
|
||||||
}
|
}
|
||||||
|
|
||||||
boolean shouldForceSplit() {
|
public Optional<byte[]> checkSplit() {
|
||||||
return this.splitRequest;
|
return checkSplit(false);
|
||||||
}
|
|
||||||
|
|
||||||
byte[] getExplicitSplitPoint() {
|
|
||||||
return this.explicitSplitPoint;
|
|
||||||
}
|
|
||||||
|
|
||||||
void forceSplit(byte[] sp) {
|
|
||||||
// This HRegion will go away after the forced split is successful
|
|
||||||
// But if a forced split fails, we need to clear forced split.
|
|
||||||
this.splitRequest = true;
|
|
||||||
if (sp != null) {
|
|
||||||
this.explicitSplitPoint = sp;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void clearSplit() {
|
|
||||||
this.splitRequest = false;
|
|
||||||
this.explicitSplitPoint = null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Return the splitpoint. null indicates the region isn't splittable
|
* Return the split point. An empty result indicates the region isn't splittable.
|
||||||
* If the splitpoint isn't explicitly specified, it will go over the stores
|
|
||||||
* to find the best splitpoint. Currently the criteria of best splitpoint
|
|
||||||
* is based on the size of the store.
|
|
||||||
*/
|
*/
|
||||||
public byte[] checkSplit() {
|
public Optional<byte[]> checkSplit(boolean force) {
|
||||||
// Can't split META
|
// Can't split META
|
||||||
if (this.getRegionInfo().isMetaRegion() ||
|
if (this.getRegionInfo().isMetaRegion() ||
|
||||||
TableName.NAMESPACE_TABLE_NAME.equals(this.getRegionInfo().getTable())) {
|
TableName.NAMESPACE_TABLE_NAME.equals(this.getRegionInfo().getTable())) {
|
||||||
if (shouldForceSplit()) {
|
return Optional.empty();
|
||||||
LOG.warn("Cannot split meta region in HBase 0.20 and above");
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Can't split a region that is closing.
|
// Can't split a region that is closing.
|
||||||
if (this.isClosing()) {
|
if (this.isClosing()) {
|
||||||
return null;
|
return Optional.empty();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!splitPolicy.shouldSplit()) {
|
if (!force && !splitPolicy.shouldSplit()) {
|
||||||
return null;
|
return Optional.empty();
|
||||||
}
|
}
|
||||||
|
|
||||||
byte[] ret = splitPolicy.getSplitPoint();
|
byte[] ret = splitPolicy.getSplitPoint();
|
||||||
|
@ -8537,10 +8512,12 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
|
||||||
checkRow(ret, "calculated split");
|
checkRow(ret, "calculated split");
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
LOG.error("Ignoring invalid split for region {}", this, e);
|
LOG.error("Ignoring invalid split for region {}", this, e);
|
||||||
return null;
|
return Optional.empty();
|
||||||
}
|
}
|
||||||
|
return Optional.of(ret);
|
||||||
|
} else {
|
||||||
|
return Optional.empty();
|
||||||
}
|
}
|
||||||
return ret;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -70,7 +70,6 @@ public class IncreasingToUpperBoundRegionSplitPolicy extends ConstantSizeRegionS
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected boolean shouldSplit() {
|
protected boolean shouldSplit() {
|
||||||
boolean force = region.shouldForceSplit();
|
|
||||||
boolean foundABigStore = false;
|
boolean foundABigStore = false;
|
||||||
// Get count of regions that have the same common table as this.region
|
// Get count of regions that have the same common table as this.region
|
||||||
int tableRegionsCount = getCountOfCommonTableRegions();
|
int tableRegionsCount = getCountOfCommonTableRegions();
|
||||||
|
@ -95,7 +94,7 @@ public class IncreasingToUpperBoundRegionSplitPolicy extends ConstantSizeRegionS
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return foundABigStore || force;
|
return foundABigStore;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -610,7 +610,7 @@ class MemStoreFlusher implements FlushRequester {
|
||||||
FlushResult flushResult = region.flushcache(forceFlushAllStores, false, tracker);
|
FlushResult flushResult = region.flushcache(forceFlushAllStores, false, tracker);
|
||||||
boolean shouldCompact = flushResult.isCompactionNeeded();
|
boolean shouldCompact = flushResult.isCompactionNeeded();
|
||||||
// We just want to check the size
|
// We just want to check the size
|
||||||
boolean shouldSplit = region.checkSplit() != null;
|
boolean shouldSplit = region.checkSplit().isPresent();
|
||||||
if (shouldSplit) {
|
if (shouldSplit) {
|
||||||
this.server.compactSplitThread.requestSplit(region);
|
this.server.compactSplitThread.requestSplit(region);
|
||||||
} else if (shouldCompact) {
|
} else if (shouldCompact) {
|
||||||
|
|
|
@ -1839,29 +1839,24 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
|
||||||
requestCount.increment();
|
requestCount.increment();
|
||||||
HRegion region = getRegion(request.getRegion());
|
HRegion region = getRegion(request.getRegion());
|
||||||
RegionInfo info = region.getRegionInfo();
|
RegionInfo info = region.getRegionInfo();
|
||||||
byte[] bestSplitRow = null;
|
byte[] bestSplitRow;
|
||||||
boolean shouldSplit = true;
|
|
||||||
if (request.hasBestSplitRow() && request.getBestSplitRow()) {
|
if (request.hasBestSplitRow() && request.getBestSplitRow()) {
|
||||||
HRegion r = region;
|
bestSplitRow = region.checkSplit(true).orElse(null);
|
||||||
region.startRegionOperation(Operation.SPLIT_REGION);
|
|
||||||
r.forceSplit(null);
|
|
||||||
// Even after setting force split if split policy says no to split then we should not split.
|
|
||||||
shouldSplit = region.getSplitPolicy().shouldSplit() && !info.isMetaRegion();
|
|
||||||
bestSplitRow = r.checkSplit();
|
|
||||||
// when all table data are in memstore, bestSplitRow = null
|
// when all table data are in memstore, bestSplitRow = null
|
||||||
// try to flush region first
|
// try to flush region first
|
||||||
if(bestSplitRow == null) {
|
if (bestSplitRow == null) {
|
||||||
r.flush(true);
|
region.flush(true);
|
||||||
bestSplitRow = r.checkSplit();
|
bestSplitRow = region.checkSplit(true).orElse(null);
|
||||||
}
|
}
|
||||||
r.clearSplit();
|
} else {
|
||||||
|
bestSplitRow = null;
|
||||||
}
|
}
|
||||||
GetRegionInfoResponse.Builder builder = GetRegionInfoResponse.newBuilder();
|
GetRegionInfoResponse.Builder builder = GetRegionInfoResponse.newBuilder();
|
||||||
builder.setRegionInfo(ProtobufUtil.toRegionInfo(info));
|
builder.setRegionInfo(ProtobufUtil.toRegionInfo(info));
|
||||||
if (request.hasCompactionState() && request.getCompactionState()) {
|
if (request.hasCompactionState() && request.getCompactionState()) {
|
||||||
builder.setCompactionState(ProtobufUtil.createCompactionState(region.getCompactionState()));
|
builder.setCompactionState(ProtobufUtil.createCompactionState(region.getCompactionState()));
|
||||||
}
|
}
|
||||||
builder.setSplittable(region.isSplittable() && shouldSplit);
|
builder.setSplittable(region.isSplittable());
|
||||||
builder.setMergeable(region.isMergeable());
|
builder.setMergeable(region.isMergeable());
|
||||||
if (request.hasBestSplitRow() && request.getBestSplitRow() && bestSplitRow != null) {
|
if (request.hasBestSplitRow() && request.getBestSplitRow() && bestSplitRow != null) {
|
||||||
builder.setBestSplitRow(UnsafeByteOperations.unsafeWrap(bestSplitRow));
|
builder.setBestSplitRow(UnsafeByteOperations.unsafeWrap(bestSplitRow));
|
||||||
|
|
|
@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.conf.Configured;
|
import org.apache.hadoop.conf.Configured;
|
||||||
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
|
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
|
import org.apache.hadoop.hbase.TableName;
|
||||||
import org.apache.hadoop.hbase.client.TableDescriptor;
|
import org.apache.hadoop.hbase.client.TableDescriptor;
|
||||||
import org.apache.hadoop.util.ReflectionUtils;
|
import org.apache.hadoop.util.ReflectionUtils;
|
||||||
import org.apache.yetus.audience.InterfaceAudience;
|
import org.apache.yetus.audience.InterfaceAudience;
|
||||||
|
@ -70,16 +71,21 @@ public abstract class RegionSplitPolicy extends Configured {
|
||||||
*/
|
*/
|
||||||
protected abstract boolean shouldSplit();
|
protected abstract boolean shouldSplit();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return {@code true} if the specified region can be split.
|
||||||
|
*/
|
||||||
|
protected boolean canSplit() {
|
||||||
|
return !region.getRegionInfo().isMetaRegion() &&
|
||||||
|
!TableName.NAMESPACE_TABLE_NAME.equals(region.getRegionInfo().getTable()) &&
|
||||||
|
region.isAvailable() && !region.hasReferences();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return the key at which the region should be split, or null
|
* @return the key at which the region should be split, or null
|
||||||
* if it cannot be split. This will only be called if shouldSplit
|
* if it cannot be split. This will only be called if shouldSplit
|
||||||
* previously returned true.
|
* previously returned true.
|
||||||
*/
|
*/
|
||||||
protected byte[] getSplitPoint() {
|
protected byte[] getSplitPoint() {
|
||||||
byte[] explicitSplitPoint = this.region.getExplicitSplitPoint();
|
|
||||||
if (explicitSplitPoint != null) {
|
|
||||||
return explicitSplitPoint;
|
|
||||||
}
|
|
||||||
List<HStore> stores = region.getStores();
|
List<HStore> stores = region.getStores();
|
||||||
|
|
||||||
byte[] splitPointFromLargestStore = null;
|
byte[] splitPointFromLargestStore = null;
|
||||||
|
|
|
@ -3247,9 +3247,8 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
|
||||||
unassignRegion(hrl.getRegionInfo().getRegionName());
|
unassignRegion(hrl.getRegionInfo().getRegionName());
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/**
|
||||||
* Retrieves a splittable region randomly from tableName
|
* Retrieves a splittable region randomly from tableName
|
||||||
*
|
|
||||||
* @param tableName name of table
|
* @param tableName name of table
|
||||||
* @param maxAttempts maximum number of attempts, unlimited for value of -1
|
* @param maxAttempts maximum number of attempts, unlimited for value of -1
|
||||||
* @return the HRegion chosen, null if none was found within limit of maxAttempts
|
* @return the HRegion chosen, null if none was found within limit of maxAttempts
|
||||||
|
@ -3272,15 +3271,14 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
|
||||||
if (regCount > 0) {
|
if (regCount > 0) {
|
||||||
idx = random.nextInt(regCount);
|
idx = random.nextInt(regCount);
|
||||||
// if we have just tried this region, there is no need to try again
|
// if we have just tried this region, there is no need to try again
|
||||||
if (attempted.contains(idx))
|
if (attempted.contains(idx)) {
|
||||||
continue;
|
continue;
|
||||||
try {
|
|
||||||
regions.get(idx).checkSplit();
|
|
||||||
return regions.get(idx);
|
|
||||||
} catch (Exception ex) {
|
|
||||||
LOG.warn("Caught exception", ex);
|
|
||||||
attempted.add(idx);
|
|
||||||
}
|
}
|
||||||
|
HRegion region = regions.get(idx);
|
||||||
|
if (region.checkSplit().isPresent()) {
|
||||||
|
return region;
|
||||||
|
}
|
||||||
|
attempted.add(idx);
|
||||||
}
|
}
|
||||||
attempts++;
|
attempts++;
|
||||||
} while (maxAttempts == -1 || attempts < maxAttempts);
|
} while (maxAttempts == -1 || attempts < maxAttempts);
|
||||||
|
|
|
@ -21,6 +21,7 @@ import static org.junit.Assert.assertArrayEquals;
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.assertEquals;
|
||||||
import static org.junit.Assert.assertFalse;
|
import static org.junit.Assert.assertFalse;
|
||||||
import static org.junit.Assert.assertTrue;
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
@ -69,6 +70,7 @@ import org.junit.experimental.categories.Category;
|
||||||
import org.junit.rules.TestName;
|
import org.junit.rules.TestName;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import org.apache.hbase.thirdparty.com.google.common.collect.Iterators;
|
import org.apache.hbase.thirdparty.com.google.common.collect.Iterators;
|
||||||
import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
|
import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
|
||||||
import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
|
import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
|
||||||
|
@ -120,7 +122,6 @@ public class TestEndToEndSplitTransaction {
|
||||||
admin.createTable(htd);
|
admin.createTable(htd);
|
||||||
TEST_UTIL.loadTable(source, fam);
|
TEST_UTIL.loadTable(source, fam);
|
||||||
compactSplit.setCompactionsEnabled(false);
|
compactSplit.setCompactionsEnabled(false);
|
||||||
TEST_UTIL.getHBaseCluster().getRegions(tableName).get(0).forceSplit(null);
|
|
||||||
admin.split(tableName);
|
admin.split(tableName);
|
||||||
TEST_UTIL.waitFor(60000, () -> TEST_UTIL.getHBaseCluster().getRegions(tableName).size() == 2);
|
TEST_UTIL.waitFor(60000, () -> TEST_UTIL.getHBaseCluster().getRegions(tableName).size() == 2);
|
||||||
|
|
||||||
|
|
|
@ -874,9 +874,6 @@ public class TestHStore {
|
||||||
public void testSplitWithEmptyColFam() throws IOException {
|
public void testSplitWithEmptyColFam() throws IOException {
|
||||||
init(this.name.getMethodName());
|
init(this.name.getMethodName());
|
||||||
assertFalse(store.getSplitPoint().isPresent());
|
assertFalse(store.getSplitPoint().isPresent());
|
||||||
store.getHRegion().forceSplit(null);
|
|
||||||
assertFalse(store.getSplitPoint().isPresent());
|
|
||||||
store.getHRegion().clearSplit();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
|
@ -21,6 +21,9 @@ import static org.junit.Assert.assertEquals;
|
||||||
import static org.junit.Assert.assertFalse;
|
import static org.junit.Assert.assertFalse;
|
||||||
import static org.junit.Assert.assertNull;
|
import static org.junit.Assert.assertNull;
|
||||||
import static org.junit.Assert.assertTrue;
|
import static org.junit.Assert.assertTrue;
|
||||||
|
import static org.mockito.Mockito.doReturn;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
@ -30,22 +33,21 @@ import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
import org.apache.hadoop.hbase.HConstants;
|
||||||
import org.apache.hadoop.hbase.HRegionInfo;
|
|
||||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
|
||||||
import org.apache.hadoop.hbase.TableName;
|
import org.apache.hadoop.hbase.TableName;
|
||||||
|
import org.apache.hadoop.hbase.client.RegionInfo;
|
||||||
|
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
|
||||||
|
import org.apache.hadoop.hbase.client.TableDescriptor;
|
||||||
|
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||||
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
|
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
|
||||||
import org.apache.hadoop.hbase.testclassification.SmallTests;
|
import org.apache.hadoop.hbase.testclassification.SmallTests;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
|
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.ClassRule;
|
import org.junit.ClassRule;
|
||||||
import org.junit.Rule;
|
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.junit.experimental.categories.Category;
|
import org.junit.experimental.categories.Category;
|
||||||
import org.junit.rules.TestName;
|
|
||||||
import org.mockito.Mockito;
|
|
||||||
|
|
||||||
@Category({RegionServerTests.class, SmallTests.class})
|
@Category({ RegionServerTests.class, SmallTests.class })
|
||||||
public class TestRegionSplitPolicy {
|
public class TestRegionSplitPolicy {
|
||||||
|
|
||||||
@ClassRule
|
@ClassRule
|
||||||
|
@ -53,52 +55,42 @@ public class TestRegionSplitPolicy {
|
||||||
HBaseClassTestRule.forClass(TestRegionSplitPolicy.class);
|
HBaseClassTestRule.forClass(TestRegionSplitPolicy.class);
|
||||||
|
|
||||||
private Configuration conf;
|
private Configuration conf;
|
||||||
private HTableDescriptor htd;
|
|
||||||
private HRegion mockRegion;
|
private HRegion mockRegion;
|
||||||
private List<HStore> stores;
|
private List<HStore> stores;
|
||||||
private static final TableName TABLENAME = TableName.valueOf("t");
|
private static final TableName TABLENAME = TableName.valueOf("t");
|
||||||
|
|
||||||
@Rule
|
|
||||||
public TestName name = new TestName();
|
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void setupMocks() {
|
public void setupMocks() {
|
||||||
conf = HBaseConfiguration.create();
|
conf = HBaseConfiguration.create();
|
||||||
HRegionInfo hri = new HRegionInfo(TABLENAME);
|
RegionInfo hri = RegionInfoBuilder.newBuilder(TABLENAME).build();
|
||||||
htd = new HTableDescriptor(TABLENAME);
|
mockRegion = mock(HRegion.class);
|
||||||
mockRegion = Mockito.mock(HRegion.class);
|
doReturn(hri).when(mockRegion).getRegionInfo();
|
||||||
Mockito.doReturn(htd).when(mockRegion).getTableDescriptor();
|
|
||||||
Mockito.doReturn(hri).when(mockRegion).getRegionInfo();
|
|
||||||
stores = new ArrayList<>();
|
stores = new ArrayList<>();
|
||||||
Mockito.doReturn(stores).when(mockRegion).getStores();
|
doReturn(stores).when(mockRegion).getStores();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testForceSplitRegionWithReference() throws IOException {
|
public void testForceSplitRegionWithReference() throws IOException {
|
||||||
htd.setMaxFileSize(1024L);
|
TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(1024L).build();
|
||||||
|
doReturn(td).when(mockRegion).getTableDescriptor();
|
||||||
// Add a store above the requisite size. Should split.
|
// Add a store above the requisite size. Should split.
|
||||||
HStore mockStore = Mockito.mock(HStore.class);
|
HStore mockStore = mock(HStore.class);
|
||||||
Mockito.doReturn(2000L).when(mockStore).getSize();
|
doReturn(2000L).when(mockStore).getSize();
|
||||||
// Act as if there's a reference file or some other reason it can't split.
|
// Act as if there's a reference file or some other reason it can't split.
|
||||||
// This should prevent splitting even though it's big enough.
|
// This should prevent splitting even though it's big enough.
|
||||||
Mockito.doReturn(false).when(mockStore).canSplit();
|
doReturn(false).when(mockStore).canSplit();
|
||||||
stores.add(mockStore);
|
stores.add(mockStore);
|
||||||
|
|
||||||
conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
|
conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
|
||||||
ConstantSizeRegionSplitPolicy.class.getName());
|
ConstantSizeRegionSplitPolicy.class.getName());
|
||||||
ConstantSizeRegionSplitPolicy policy =
|
ConstantSizeRegionSplitPolicy policy =
|
||||||
(ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
|
(ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
|
||||||
assertFalse(policy.shouldSplit());
|
|
||||||
Mockito.doReturn(true).when(mockRegion).shouldForceSplit();
|
|
||||||
assertFalse(policy.shouldSplit());
|
assertFalse(policy.shouldSplit());
|
||||||
|
|
||||||
Mockito.doReturn(false).when(mockRegion).shouldForceSplit();
|
|
||||||
conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
|
conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
|
||||||
IncreasingToUpperBoundRegionSplitPolicy.class.getName());
|
IncreasingToUpperBoundRegionSplitPolicy.class.getName());
|
||||||
policy = (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
|
policy = (IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
|
||||||
assertFalse(policy.shouldSplit());
|
assertFalse(policy.shouldSplit());
|
||||||
Mockito.doReturn(true).when(mockRegion).shouldForceSplit();
|
|
||||||
assertFalse(policy.shouldSplit());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -108,31 +100,32 @@ public class TestRegionSplitPolicy {
|
||||||
IncreasingToUpperBoundRegionSplitPolicy.class.getName());
|
IncreasingToUpperBoundRegionSplitPolicy.class.getName());
|
||||||
// Now make it so the mock region has a RegionServerService that will
|
// Now make it so the mock region has a RegionServerService that will
|
||||||
// return 'online regions'.
|
// return 'online regions'.
|
||||||
RegionServerServices rss = Mockito.mock(RegionServerServices.class);
|
RegionServerServices rss = mock(RegionServerServices.class);
|
||||||
final List<HRegion> regions = new ArrayList<>();
|
final List<HRegion> regions = new ArrayList<>();
|
||||||
Mockito.doReturn(regions).when(rss).getRegions(TABLENAME);
|
doReturn(regions).when(rss).getRegions(TABLENAME);
|
||||||
Mockito.when(mockRegion.getRegionServerServices()).thenReturn(rss);
|
when(mockRegion.getRegionServerServices()).thenReturn(rss);
|
||||||
// Set max size for this 'table'.
|
// Set max size for this 'table'.
|
||||||
long maxSplitSize = 1024L;
|
long maxSplitSize = 1024L;
|
||||||
htd.setMaxFileSize(maxSplitSize);
|
|
||||||
// Set flush size to 1/8. IncreasingToUpperBoundRegionSplitPolicy
|
// Set flush size to 1/8. IncreasingToUpperBoundRegionSplitPolicy
|
||||||
// grows by the cube of the number of regions times flushsize each time.
|
// grows by the cube of the number of regions times flushsize each time.
|
||||||
long flushSize = maxSplitSize/8;
|
long flushSize = maxSplitSize / 8;
|
||||||
conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSize);
|
conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, flushSize);
|
||||||
htd.setMemStoreFlushSize(flushSize);
|
TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(maxSplitSize)
|
||||||
|
.setMemStoreFlushSize(flushSize).build();
|
||||||
|
doReturn(td).when(mockRegion).getTableDescriptor();
|
||||||
// If RegionServerService with no regions in it -- 'online regions' == 0 --
|
// If RegionServerService with no regions in it -- 'online regions' == 0 --
|
||||||
// then IncreasingToUpperBoundRegionSplitPolicy should act like a
|
// then IncreasingToUpperBoundRegionSplitPolicy should act like a
|
||||||
// ConstantSizePolicy
|
// ConstantSizePolicy
|
||||||
IncreasingToUpperBoundRegionSplitPolicy policy =
|
IncreasingToUpperBoundRegionSplitPolicy policy =
|
||||||
(IncreasingToUpperBoundRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
|
(IncreasingToUpperBoundRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
|
||||||
doConstantSizePolicyTests(policy);
|
doConstantSizePolicyTests(policy);
|
||||||
|
|
||||||
// Add a store in excess of split size. Because there are "no regions"
|
// Add a store in excess of split size. Because there are "no regions"
|
||||||
// on this server -- rss.getOnlineRegions is 0 -- then we should split
|
// on this server -- rss.getOnlineRegions is 0 -- then we should split
|
||||||
// like a constantsizeregionsplitpolicy would
|
// like a constantsizeregionsplitpolicy would
|
||||||
HStore mockStore = Mockito.mock(HStore.class);
|
HStore mockStore = mock(HStore.class);
|
||||||
Mockito.doReturn(2000L).when(mockStore).getSize();
|
doReturn(2000L).when(mockStore).getSize();
|
||||||
Mockito.doReturn(true).when(mockStore).canSplit();
|
doReturn(true).when(mockStore).canSplit();
|
||||||
stores.add(mockStore);
|
stores.add(mockStore);
|
||||||
// It should split
|
// It should split
|
||||||
assertTrue(policy.shouldSplit());
|
assertTrue(policy.shouldSplit());
|
||||||
|
@ -140,18 +133,18 @@ public class TestRegionSplitPolicy {
|
||||||
// Now test that we increase our split size as online regions for a table
|
// Now test that we increase our split size as online regions for a table
|
||||||
// grows. With one region, split size should be flushsize.
|
// grows. With one region, split size should be flushsize.
|
||||||
regions.add(mockRegion);
|
regions.add(mockRegion);
|
||||||
Mockito.doReturn(flushSize).when(mockStore).getSize();
|
doReturn(flushSize).when(mockStore).getSize();
|
||||||
// Should not split since store is flush size.
|
// Should not split since store is flush size.
|
||||||
assertFalse(policy.shouldSplit());
|
assertFalse(policy.shouldSplit());
|
||||||
// Set size of store to be > 2*flush size and we should split
|
// Set size of store to be > 2*flush size and we should split
|
||||||
Mockito.doReturn(flushSize*2 + 1).when(mockStore).getSize();
|
doReturn(flushSize * 2 + 1).when(mockStore).getSize();
|
||||||
assertTrue(policy.shouldSplit());
|
assertTrue(policy.shouldSplit());
|
||||||
// Add another region to the 'online regions' on this server and we should
|
// Add another region to the 'online regions' on this server and we should
|
||||||
// now be no longer be splittable since split size has gone up.
|
// now be no longer be splittable since split size has gone up.
|
||||||
regions.add(mockRegion);
|
regions.add(mockRegion);
|
||||||
assertFalse(policy.shouldSplit());
|
assertFalse(policy.shouldSplit());
|
||||||
// make sure its just over; verify it'll split
|
// make sure its just over; verify it'll split
|
||||||
Mockito.doReturn((long)(maxSplitSize * 1.25 + 1)).when(mockStore).getSize();
|
doReturn((long) (maxSplitSize * 1.25 + 1)).when(mockStore).getSize();
|
||||||
assertTrue(policy.shouldSplit());
|
assertTrue(policy.shouldSplit());
|
||||||
|
|
||||||
// Finally assert that even if loads of regions, we'll split at max size
|
// Finally assert that even if loads of regions, we'll split at max size
|
||||||
|
@ -162,41 +155,39 @@ public class TestRegionSplitPolicy {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testBusyRegionSplitPolicy() throws Exception {
|
public void testBusyRegionSplitPolicy() throws Exception {
|
||||||
conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
|
doReturn(TableDescriptorBuilder.newBuilder(TABLENAME).build()).when(mockRegion)
|
||||||
BusyRegionSplitPolicy.class.getName());
|
.getTableDescriptor();
|
||||||
|
conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, BusyRegionSplitPolicy.class.getName());
|
||||||
conf.setLong("hbase.busy.policy.minAge", 1000000L);
|
conf.setLong("hbase.busy.policy.minAge", 1000000L);
|
||||||
conf.setFloat("hbase.busy.policy.blockedRequests", 0.1f);
|
conf.setFloat("hbase.busy.policy.blockedRequests", 0.1f);
|
||||||
|
|
||||||
RegionServerServices rss = Mockito.mock(RegionServerServices.class);
|
RegionServerServices rss = mock(RegionServerServices.class);
|
||||||
final List<HRegion> regions = new ArrayList<>();
|
final List<HRegion> regions = new ArrayList<>();
|
||||||
Mockito.doReturn(regions).when(rss).getRegions(TABLENAME);
|
doReturn(regions).when(rss).getRegions(TABLENAME);
|
||||||
Mockito.when(mockRegion.getRegionServerServices()).thenReturn(rss);
|
when(mockRegion.getRegionServerServices()).thenReturn(rss);
|
||||||
Mockito.when(mockRegion.getBlockedRequestsCount()).thenReturn(0L);
|
when(mockRegion.getBlockedRequestsCount()).thenReturn(0L);
|
||||||
Mockito.when(mockRegion.getWriteRequestsCount()).thenReturn(0L);
|
when(mockRegion.getWriteRequestsCount()).thenReturn(0L);
|
||||||
|
|
||||||
|
|
||||||
BusyRegionSplitPolicy policy =
|
BusyRegionSplitPolicy policy =
|
||||||
(BusyRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
|
(BusyRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
|
||||||
|
|
||||||
Mockito.when(mockRegion.getBlockedRequestsCount()).thenReturn(10L);
|
when(mockRegion.getBlockedRequestsCount()).thenReturn(10L);
|
||||||
Mockito.when(mockRegion.getWriteRequestsCount()).thenReturn(10L);
|
when(mockRegion.getWriteRequestsCount()).thenReturn(10L);
|
||||||
// Not enough time since region came online
|
// Not enough time since region came online
|
||||||
assertFalse(policy.shouldSplit());
|
assertFalse(policy.shouldSplit());
|
||||||
|
|
||||||
|
|
||||||
// Reset min age for split to zero
|
// Reset min age for split to zero
|
||||||
conf.setLong("hbase.busy.policy.minAge", 0L);
|
conf.setLong("hbase.busy.policy.minAge", 0L);
|
||||||
// Aggregate over 500 ms periods
|
// Aggregate over 500 ms periods
|
||||||
conf.setLong("hbase.busy.policy.aggWindow", 500L);
|
conf.setLong("hbase.busy.policy.aggWindow", 500L);
|
||||||
policy =
|
policy = (BusyRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
|
||||||
(BusyRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
|
|
||||||
long start = EnvironmentEdgeManager.currentTime();
|
long start = EnvironmentEdgeManager.currentTime();
|
||||||
Mockito.when(mockRegion.getBlockedRequestsCount()).thenReturn(10L);
|
when(mockRegion.getBlockedRequestsCount()).thenReturn(10L);
|
||||||
Mockito.when(mockRegion.getWriteRequestsCount()).thenReturn(20L);
|
when(mockRegion.getWriteRequestsCount()).thenReturn(20L);
|
||||||
Thread.sleep(300);
|
Thread.sleep(300);
|
||||||
assertFalse(policy.shouldSplit());
|
assertFalse(policy.shouldSplit());
|
||||||
Mockito.when(mockRegion.getBlockedRequestsCount()).thenReturn(12L);
|
when(mockRegion.getBlockedRequestsCount()).thenReturn(12L);
|
||||||
Mockito.when(mockRegion.getWriteRequestsCount()).thenReturn(30L);
|
when(mockRegion.getWriteRequestsCount()).thenReturn(30L);
|
||||||
Thread.sleep(2);
|
Thread.sleep(2);
|
||||||
// Enough blocked requests since last time, but aggregate blocked request
|
// Enough blocked requests since last time, but aggregate blocked request
|
||||||
// rate over last 500 ms is still low, because major portion of the window is constituted
|
// rate over last 500 ms is still low, because major portion of the window is constituted
|
||||||
|
@ -204,34 +195,33 @@ public class TestRegionSplitPolicy {
|
||||||
if (EnvironmentEdgeManager.currentTime() - start < 500) {
|
if (EnvironmentEdgeManager.currentTime() - start < 500) {
|
||||||
assertFalse(policy.shouldSplit());
|
assertFalse(policy.shouldSplit());
|
||||||
}
|
}
|
||||||
Mockito.when(mockRegion.getBlockedRequestsCount()).thenReturn(14L);
|
when(mockRegion.getBlockedRequestsCount()).thenReturn(14L);
|
||||||
Mockito.when(mockRegion.getWriteRequestsCount()).thenReturn(40L);
|
when(mockRegion.getWriteRequestsCount()).thenReturn(40L);
|
||||||
Thread.sleep(200);
|
Thread.sleep(200);
|
||||||
assertTrue(policy.shouldSplit());
|
assertTrue(policy.shouldSplit());
|
||||||
}
|
}
|
||||||
|
|
||||||
private void assertWithinJitter(long maxSplitSize, long sizeToCheck) {
|
private void assertWithinJitter(long maxSplitSize, long sizeToCheck) {
|
||||||
assertTrue("Size greater than lower bound of jitter",
|
assertTrue("Size greater than lower bound of jitter",
|
||||||
(long)(maxSplitSize * 0.75) <= sizeToCheck);
|
(long) (maxSplitSize * 0.75) <= sizeToCheck);
|
||||||
assertTrue("Size less than upper bound of jitter",
|
assertTrue("Size less than upper bound of jitter", (long) (maxSplitSize * 1.25) >= sizeToCheck);
|
||||||
(long)(maxSplitSize * 1.25) >= sizeToCheck);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testCreateDefault() throws IOException {
|
public void testCreateDefault() throws IOException {
|
||||||
conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1234L);
|
conf.setLong(HConstants.HREGION_MAX_FILESIZE, 1234L);
|
||||||
|
TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).build();
|
||||||
|
doReturn(td).when(mockRegion).getTableDescriptor();
|
||||||
// Using a default HTD, should pick up the file size from
|
// Using a default HTD, should pick up the file size from
|
||||||
// configuration.
|
// configuration.
|
||||||
ConstantSizeRegionSplitPolicy policy =
|
ConstantSizeRegionSplitPolicy policy =
|
||||||
(ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(
|
(ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
|
||||||
mockRegion, conf);
|
|
||||||
assertWithinJitter(1234L, policy.getDesiredMaxFileSize());
|
assertWithinJitter(1234L, policy.getDesiredMaxFileSize());
|
||||||
|
|
||||||
// If specified in HTD, should use that
|
// If specified in HTD, should use that
|
||||||
htd.setMaxFileSize(9999L);
|
td = TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(9999L).build();
|
||||||
policy = (ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(
|
doReturn(td).when(mockRegion).getTableDescriptor();
|
||||||
mockRegion, conf);
|
policy = (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
|
||||||
assertWithinJitter(9999L, policy.getDesiredMaxFileSize());
|
assertWithinJitter(9999L, policy.getDesiredMaxFileSize());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -240,75 +230,58 @@ public class TestRegionSplitPolicy {
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void testCustomPolicy() throws IOException {
|
public void testCustomPolicy() throws IOException {
|
||||||
HTableDescriptor myHtd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
|
TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME)
|
||||||
myHtd.setValue(HTableDescriptor.SPLIT_POLICY,
|
.setRegionSplitPolicyClassName(KeyPrefixRegionSplitPolicy.class.getName())
|
||||||
KeyPrefixRegionSplitPolicy.class.getName());
|
.setValue(KeyPrefixRegionSplitPolicy.PREFIX_LENGTH_KEY, "2").build();
|
||||||
myHtd.setValue(KeyPrefixRegionSplitPolicy.PREFIX_LENGTH_KEY, String.valueOf(2));
|
|
||||||
|
|
||||||
HRegion myMockRegion = Mockito.mock(HRegion.class);
|
doReturn(td).when(mockRegion).getTableDescriptor();
|
||||||
Mockito.doReturn(myHtd).when(myMockRegion).getTableDescriptor();
|
|
||||||
Mockito.doReturn(stores).when(myMockRegion).getStores();
|
|
||||||
|
|
||||||
HStore mockStore = Mockito.mock(HStore.class);
|
HStore mockStore = mock(HStore.class);
|
||||||
Mockito.doReturn(2000L).when(mockStore).getSize();
|
doReturn(2000L).when(mockStore).getSize();
|
||||||
Mockito.doReturn(true).when(mockStore).canSplit();
|
doReturn(true).when(mockStore).canSplit();
|
||||||
Mockito.doReturn(Optional.of(Bytes.toBytes("abcd"))).when(mockStore).getSplitPoint();
|
doReturn(Optional.of(Bytes.toBytes("abcd"))).when(mockStore).getSplitPoint();
|
||||||
stores.add(mockStore);
|
stores.add(mockStore);
|
||||||
|
|
||||||
KeyPrefixRegionSplitPolicy policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy
|
KeyPrefixRegionSplitPolicy policy =
|
||||||
.create(myMockRegion, conf);
|
(KeyPrefixRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
|
||||||
|
|
||||||
assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
|
assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
|
||||||
|
|
||||||
Mockito.doReturn(true).when(myMockRegion).shouldForceSplit();
|
|
||||||
Mockito.doReturn(Bytes.toBytes("efgh")).when(myMockRegion)
|
|
||||||
.getExplicitSplitPoint();
|
|
||||||
|
|
||||||
policy = (KeyPrefixRegionSplitPolicy) RegionSplitPolicy
|
|
||||||
.create(myMockRegion, conf);
|
|
||||||
|
|
||||||
assertEquals("ef", Bytes.toString(policy.getSplitPoint()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testConstantSizePolicy() throws IOException {
|
public void testConstantSizePolicy() throws IOException {
|
||||||
htd.setMaxFileSize(1024L);
|
TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(1024L).build();
|
||||||
|
doReturn(td).when(mockRegion).getTableDescriptor();
|
||||||
ConstantSizeRegionSplitPolicy policy =
|
ConstantSizeRegionSplitPolicy policy =
|
||||||
(ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
|
(ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
|
||||||
doConstantSizePolicyTests(policy);
|
doConstantSizePolicyTests(policy);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Run through tests for a ConstantSizeRegionSplitPolicy
|
* Run through tests for a ConstantSizeRegionSplitPolicy
|
||||||
* @param policy
|
|
||||||
*/
|
*/
|
||||||
private void doConstantSizePolicyTests(final ConstantSizeRegionSplitPolicy policy) {
|
private void doConstantSizePolicyTests(final ConstantSizeRegionSplitPolicy policy) {
|
||||||
// For no stores, should not split
|
// For no stores, should not split
|
||||||
assertFalse(policy.shouldSplit());
|
assertFalse(policy.shouldSplit());
|
||||||
|
|
||||||
// Add a store above the requisite size. Should split.
|
// Add a store above the requisite size. Should split.
|
||||||
HStore mockStore = Mockito.mock(HStore.class);
|
HStore mockStore = mock(HStore.class);
|
||||||
Mockito.doReturn(2000L).when(mockStore).getSize();
|
doReturn(2000L).when(mockStore).getSize();
|
||||||
Mockito.doReturn(true).when(mockStore).canSplit();
|
doReturn(true).when(mockStore).canSplit();
|
||||||
stores.add(mockStore);
|
stores.add(mockStore);
|
||||||
|
|
||||||
assertTrue(policy.shouldSplit());
|
assertTrue(policy.shouldSplit());
|
||||||
|
|
||||||
// Act as if there's a reference file or some other reason it can't split.
|
// Act as if there's a reference file or some other reason it can't split.
|
||||||
// This should prevent splitting even though it's big enough.
|
// This should prevent splitting even though it's big enough.
|
||||||
Mockito.doReturn(false).when(mockStore).canSplit();
|
doReturn(false).when(mockStore).canSplit();
|
||||||
assertFalse(policy.shouldSplit());
|
assertFalse(policy.shouldSplit());
|
||||||
|
|
||||||
// Reset splittability after above
|
// Reset splittability after above
|
||||||
Mockito.doReturn(true).when(mockStore).canSplit();
|
doReturn(true).when(mockStore).canSplit();
|
||||||
|
|
||||||
// Set to a small size but turn on forceSplit. Should result in a split.
|
// Set to a small size, should not split
|
||||||
Mockito.doReturn(true).when(mockRegion).shouldForceSplit();
|
doReturn(100L).when(mockStore).getSize();
|
||||||
Mockito.doReturn(100L).when(mockStore).getSize();
|
|
||||||
assertTrue(policy.shouldSplit());
|
|
||||||
|
|
||||||
// Turn off forceSplit, should not split
|
|
||||||
Mockito.doReturn(false).when(mockRegion).shouldForceSplit();
|
|
||||||
assertFalse(policy.shouldSplit());
|
assertFalse(policy.shouldSplit());
|
||||||
|
|
||||||
// Clear families we added above
|
// Clear families we added above
|
||||||
|
@ -317,67 +290,56 @@ public class TestRegionSplitPolicy {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testGetSplitPoint() throws IOException {
|
public void testGetSplitPoint() throws IOException {
|
||||||
|
TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME).build();
|
||||||
|
doReturn(td).when(mockRegion).getTableDescriptor();
|
||||||
|
|
||||||
ConstantSizeRegionSplitPolicy policy =
|
ConstantSizeRegionSplitPolicy policy =
|
||||||
(ConstantSizeRegionSplitPolicy)RegionSplitPolicy.create(mockRegion, conf);
|
(ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
|
||||||
|
|
||||||
// For no stores, should not split
|
// For no stores, should not split
|
||||||
assertFalse(policy.shouldSplit());
|
assertFalse(policy.shouldSplit());
|
||||||
assertNull(policy.getSplitPoint());
|
assertNull(policy.getSplitPoint());
|
||||||
|
|
||||||
// Add a store above the requisite size. Should split.
|
// Add a store above the requisite size. Should split.
|
||||||
HStore mockStore = Mockito.mock(HStore.class);
|
HStore mockStore = mock(HStore.class);
|
||||||
Mockito.doReturn(2000L).when(mockStore).getSize();
|
doReturn(2000L).when(mockStore).getSize();
|
||||||
Mockito.doReturn(true).when(mockStore).canSplit();
|
doReturn(true).when(mockStore).canSplit();
|
||||||
Mockito.doReturn(Optional.of(Bytes.toBytes("store 1 split"))).when(mockStore).getSplitPoint();
|
doReturn(Optional.of(Bytes.toBytes("store 1 split"))).when(mockStore).getSplitPoint();
|
||||||
stores.add(mockStore);
|
stores.add(mockStore);
|
||||||
|
|
||||||
assertEquals("store 1 split",
|
assertEquals("store 1 split", Bytes.toString(policy.getSplitPoint()));
|
||||||
Bytes.toString(policy.getSplitPoint()));
|
|
||||||
|
|
||||||
// Add a bigger store. The split point should come from that one
|
// Add a bigger store. The split point should come from that one
|
||||||
HStore mockStore2 = Mockito.mock(HStore.class);
|
HStore mockStore2 = mock(HStore.class);
|
||||||
Mockito.doReturn(4000L).when(mockStore2).getSize();
|
doReturn(4000L).when(mockStore2).getSize();
|
||||||
Mockito.doReturn(true).when(mockStore2).canSplit();
|
doReturn(true).when(mockStore2).canSplit();
|
||||||
Mockito.doReturn(Optional.of(Bytes.toBytes("store 2 split"))).when(mockStore2).getSplitPoint();
|
doReturn(Optional.of(Bytes.toBytes("store 2 split"))).when(mockStore2).getSplitPoint();
|
||||||
stores.add(mockStore2);
|
stores.add(mockStore2);
|
||||||
|
|
||||||
assertEquals("store 2 split",
|
assertEquals("store 2 split", Bytes.toString(policy.getSplitPoint()));
|
||||||
Bytes.toString(policy.getSplitPoint()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testDelimitedKeyPrefixRegionSplitPolicy() throws IOException {
|
public void testDelimitedKeyPrefixRegionSplitPolicy() throws IOException {
|
||||||
HTableDescriptor myHtd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
|
TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLENAME)
|
||||||
myHtd.setValue(HTableDescriptor.SPLIT_POLICY,
|
.setRegionSplitPolicyClassName(DelimitedKeyPrefixRegionSplitPolicy.class.getName())
|
||||||
DelimitedKeyPrefixRegionSplitPolicy.class.getName());
|
.setValue(DelimitedKeyPrefixRegionSplitPolicy.DELIMITER_KEY, ",").build();
|
||||||
myHtd.setValue(DelimitedKeyPrefixRegionSplitPolicy.DELIMITER_KEY, ",");
|
|
||||||
|
|
||||||
HRegion myMockRegion = Mockito.mock(HRegion.class);
|
doReturn(td).when(mockRegion).getTableDescriptor();
|
||||||
Mockito.doReturn(myHtd).when(myMockRegion).getTableDescriptor();
|
doReturn(stores).when(mockRegion).getStores();
|
||||||
Mockito.doReturn(stores).when(myMockRegion).getStores();
|
|
||||||
|
|
||||||
HStore mockStore = Mockito.mock(HStore.class);
|
HStore mockStore = mock(HStore.class);
|
||||||
Mockito.doReturn(2000L).when(mockStore).getSize();
|
doReturn(2000L).when(mockStore).getSize();
|
||||||
Mockito.doReturn(true).when(mockStore).canSplit();
|
doReturn(true).when(mockStore).canSplit();
|
||||||
Mockito.doReturn(Optional.of(Bytes.toBytes("ab,cd"))).when(mockStore).getSplitPoint();
|
doReturn(Optional.of(Bytes.toBytes("ab,cd"))).when(mockStore).getSplitPoint();
|
||||||
stores.add(mockStore);
|
stores.add(mockStore);
|
||||||
|
|
||||||
DelimitedKeyPrefixRegionSplitPolicy policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy
|
DelimitedKeyPrefixRegionSplitPolicy policy =
|
||||||
.create(myMockRegion, conf);
|
(DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
|
||||||
|
|
||||||
assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
|
assertEquals("ab", Bytes.toString(policy.getSplitPoint()));
|
||||||
|
|
||||||
Mockito.doReturn(true).when(myMockRegion).shouldForceSplit();
|
doReturn(Optional.of(Bytes.toBytes("ijk"))).when(mockStore).getSplitPoint();
|
||||||
Mockito.doReturn(Bytes.toBytes("efg,h")).when(myMockRegion)
|
|
||||||
.getExplicitSplitPoint();
|
|
||||||
|
|
||||||
policy = (DelimitedKeyPrefixRegionSplitPolicy) RegionSplitPolicy
|
|
||||||
.create(myMockRegion, conf);
|
|
||||||
|
|
||||||
assertEquals("efg", Bytes.toString(policy.getSplitPoint()));
|
|
||||||
|
|
||||||
Mockito.doReturn(Bytes.toBytes("ijk")).when(myMockRegion)
|
|
||||||
.getExplicitSplitPoint();
|
|
||||||
assertEquals("ijk", Bytes.toString(policy.getSplitPoint()));
|
assertEquals("ijk", Bytes.toString(policy.getSplitPoint()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -385,7 +347,9 @@ public class TestRegionSplitPolicy {
|
||||||
public void testConstantSizePolicyWithJitter() throws IOException {
|
public void testConstantSizePolicyWithJitter() throws IOException {
|
||||||
conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
|
conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
|
||||||
ConstantSizeRegionSplitPolicy.class.getName());
|
ConstantSizeRegionSplitPolicy.class.getName());
|
||||||
htd.setMaxFileSize(Long.MAX_VALUE);
|
TableDescriptor td =
|
||||||
|
TableDescriptorBuilder.newBuilder(TABLENAME).setMaxFileSize(Long.MAX_VALUE).build();
|
||||||
|
doReturn(td).when(mockRegion).getTableDescriptor();
|
||||||
boolean positiveJitter = false;
|
boolean positiveJitter = false;
|
||||||
ConstantSizeRegionSplitPolicy policy = null;
|
ConstantSizeRegionSplitPolicy policy = null;
|
||||||
while (!positiveJitter) {
|
while (!positiveJitter) {
|
||||||
|
@ -393,12 +357,11 @@ public class TestRegionSplitPolicy {
|
||||||
positiveJitter = policy.positiveJitterRate();
|
positiveJitter = policy.positiveJitterRate();
|
||||||
}
|
}
|
||||||
// add a store
|
// add a store
|
||||||
HStore mockStore = Mockito.mock(HStore.class);
|
HStore mockStore = mock(HStore.class);
|
||||||
Mockito.doReturn(2000L).when(mockStore).getSize();
|
doReturn(2000L).when(mockStore).getSize();
|
||||||
Mockito.doReturn(true).when(mockStore).canSplit();
|
doReturn(true).when(mockStore).canSplit();
|
||||||
stores.add(mockStore);
|
stores.add(mockStore);
|
||||||
// Jitter shouldn't cause overflow when HTableDescriptor.MAX_FILESIZE set to Long.MAX_VALUE
|
// Jitter shouldn't cause overflow when HTableDescriptor.MAX_FILESIZE set to Long.MAX_VALUE
|
||||||
assertFalse(policy.shouldSplit());
|
assertFalse(policy.shouldSplit());
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue