HBASE-15324 Jitter may cause desiredMaxFileSize overflow in ConstantSizeRegionSplitPolicy and trigger unexpected split (Yu Li)

This commit is contained in:
stack 2016-03-30 13:31:09 -07:00
parent 21301a8a95
commit 9d56105eec
2 changed files with 40 additions and 4 deletions

View File

@ -17,13 +17,15 @@
*/
package org.apache.hadoop.hbase.regionserver;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import com.google.common.annotations.VisibleForTesting;
import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import java.util.Random;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
/**
* A {@link RegionSplitPolicy} implementation which splits a region
@ -37,8 +39,10 @@ import java.util.Random;
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
public class ConstantSizeRegionSplitPolicy extends RegionSplitPolicy {
private static final Random RANDOM = new Random();
private static final double EPSILON = 1E-6;
private long desiredMaxFileSize;
private double jitterRate;
@Override
protected void configureForRegion(HRegion region) {
@ -53,7 +57,14 @@ public class ConstantSizeRegionSplitPolicy extends RegionSplitPolicy {
HConstants.DEFAULT_MAX_FILE_SIZE);
}
double jitter = conf.getDouble("hbase.hregion.max.filesize.jitter", 0.25D);
this.desiredMaxFileSize += (long)(desiredMaxFileSize * (RANDOM.nextFloat() - 0.5D) * jitter);
this.jitterRate = (RANDOM.nextFloat() - 0.5D) * jitter;
long jitterValue = (long) (this.desiredMaxFileSize * this.jitterRate);
// make sure the long value won't overflow with jitter
if (this.jitterRate > EPSILON && jitterValue > (Long.MAX_VALUE - this.desiredMaxFileSize)) {
this.desiredMaxFileSize = Long.MAX_VALUE;
} else {
this.desiredMaxFileSize += jitterValue;
}
}
@Override
@ -80,4 +91,9 @@ public class ConstantSizeRegionSplitPolicy extends RegionSplitPolicy {
long getDesiredMaxFileSize() {
return desiredMaxFileSize;
}
@VisibleForTesting
public boolean positiveJitterRate() {
return this.jitterRate > EPSILON;
}
}

View File

@ -321,4 +321,24 @@ public class TestRegionSplitPolicy {
assertEquals("ijk", Bytes.toString(policy.getSplitPoint()));
}
@Test
public void testConstantSizePolicyWithJitter() throws IOException {
conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
ConstantSizeRegionSplitPolicy.class.getName());
htd.setMaxFileSize(Long.MAX_VALUE);
boolean positiveJitter = false;
ConstantSizeRegionSplitPolicy policy = null;
while (!positiveJitter) {
policy = (ConstantSizeRegionSplitPolicy) RegionSplitPolicy.create(mockRegion, conf);
positiveJitter = policy.positiveJitterRate();
}
// add a store
HStore mockStore = Mockito.mock(HStore.class);
Mockito.doReturn(2000L).when(mockStore).getSize();
Mockito.doReturn(true).when(mockStore).canSplit();
stores.add(mockStore);
// Jitter shouldn't cause overflow when HTableDescriptor.MAX_FILESIZE set to Long.MAX_VALUE
assertFalse(policy.shouldSplit());
}
}