Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-22980 HRegionPartioner getPartition() method incorrectly partitions the regions of the table. (#590) 

Signed-off-by: Guangxu Cheng <guangxucheng@gmail.com>
This commit is contained in:
Shardul Singh 2019-11-06 19:37:44 +05:30 committed by Guangxu Cheng
parent d2142a8ebb
commit f58bd4a7ac
3 changed files with 26 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapred/HRegionPartitioner.java
View File

@ -82,7 +82,7 @@ implements Partitioner<ImmutableBytesWritable, V2> {
} }
for (int i = 0; i < this.startKeys.length; i++){ for (int i = 0; i < this.startKeys.length; i++){
if (Bytes.compareTo(region, this.startKeys[i]) == 0 ){ if (Bytes.compareTo(region, this.startKeys[i]) == 0 ){
if (i >= numPartitions-1){ if (i >= numPartitions){
// cover if we have less reduces then regions. // cover if we have less reduces then regions.
return (Integer.toString(i).hashCode() return (Integer.toString(i).hashCode()
& Integer.MAX_VALUE) % numPartitions; & Integer.MAX_VALUE) % numPartitions;

2
hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HRegionPartitioner.java
View File

@ -89,7 +89,7 @@ implements Configurable {
} }
for (int i = 0; i < this.startKeys.length; i++){ for (int i = 0; i < this.startKeys.length; i++){
if (Bytes.compareTo(region, this.startKeys[i]) == 0 ){ if (Bytes.compareTo(region, this.startKeys[i]) == 0 ){
if (i >= numPartitions-1){ if (i >= numPartitions){
// cover if we have less reduces then regions. // cover if we have less reduces then regions.
return (Integer.toString(i).hashCode() return (Integer.toString(i).hashCode()
& Integer.MAX_VALUE) % numPartitions; & Integer.MAX_VALUE) % numPartitions;

24
hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java
View File

@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.testclassification.MapReduceTests; import org.apache.hadoop.hbase.testclassification.MapReduceTests;
@ -77,4 +78,27 @@ public class TestHRegionPartitioner {
assertEquals(1, partitioner.getPartition(writable, 10L, 3)); assertEquals(1, partitioner.getPartition(writable, 10L, 3));
assertEquals(0, partitioner.getPartition(writable, 10L, 1)); assertEquals(0, partitioner.getPartition(writable, 10L, 1));
} }
@Test
public void testHRegionPartitionerMoreRegions() throws Exception {
byte[][] families = { Bytes.toBytes("familyA"), Bytes.toBytes("familyB") };
TableName tableName = TableName.valueOf(name.getMethodName());
UTIL.createTable(tableName, families, 1, Bytes.toBytes("aa"), Bytes.toBytes("cc"), 5);
Configuration configuration = UTIL.getConfiguration();
int numberOfRegions = MetaTableAccessor.getRegionCount(configuration, tableName);
assertEquals(5, numberOfRegions);
HRegionPartitioner<Long, Long> partitioner = new HRegionPartitioner<>();
configuration.set(TableOutputFormat.OUTPUT_TABLE, name.getMethodName());
partitioner.setConf(configuration);
// Get some rowKey for the lastRegion
ImmutableBytesWritable writable = new ImmutableBytesWritable(Bytes.toBytes("df"));
// getPartition should return 4 since number of partition = number of reduces.
assertEquals(4, partitioner.getPartition(writable, 10L, 5));
}
} }