diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java index 960b91fa158..19a5e03b72e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -64,6 +64,10 @@ public class HTableDescriptor implements TableDescriptor, Comparable 0) { + avgRegionSize = targetRegionSize; + } else if (targetRegionCount > 0) { + avgRegionSize = totalSizeMb / (double) targetRegionCount; + } else { + avgRegionSize = acutalRegionCnt == 0 ? 0 : totalSizeMb / (double) acutalRegionCnt; + } LOG.debug("Table " + table + ", total aggregated regions size: " + totalSizeMb); LOG.debug("Table " + table + ", average region size: " + avgRegionSize); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java index bcc89b8e514..d2123fb44f6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Rule; @@ -350,6 +351,114 @@ public class TestSimpleRegionNormalizer { assertEquals(hri4, ((SplitNormalizationPlan) plan).getRegionInfo()); } + @Test + public void testSplitWithTargetRegionCount() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + List RegionInfo = new ArrayList<>(); + Map regionSizes = new HashMap<>(); + + RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("aaa")) + .setEndKey(Bytes.toBytes("bbb")).build(); + RegionInfo.add(hri1); + regionSizes.put(hri1.getRegionName(), 20); + + RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("bbb")) + .setEndKey(Bytes.toBytes("ccc")).build(); + RegionInfo.add(hri2); + regionSizes.put(hri2.getRegionName(), 40); + + RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("ccc")) + .setEndKey(Bytes.toBytes("ddd")).build(); + RegionInfo.add(hri3); + regionSizes.put(hri3.getRegionName(), 60); + + RegionInfo hri4 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("ddd")) + .setEndKey(Bytes.toBytes("eee")).build(); + RegionInfo.add(hri4); + regionSizes.put(hri4.getRegionName(), 80); + + RegionInfo hri5 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("eee")) + .setEndKey(Bytes.toBytes("fff")).build(); + RegionInfo.add(hri5); + regionSizes.put(hri5.getRegionName(), 100); + + RegionInfo hri6 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("fff")) + .setEndKey(Bytes.toBytes("ggg")).build(); + RegionInfo.add(hri6); + regionSizes.put(hri6.getRegionName(), 120); + + setupMocksForNormalizer(regionSizes, RegionInfo); + + // test when target region size is 20 + when(masterServices.getTableDescriptors().get(any()).getNormalizerTargetRegionSize()) + .thenReturn(20L); + List plans = normalizer.computePlanForTable(tableName); + Assert.assertEquals(4, plans.size()); + + for (NormalizationPlan plan : plans) { + assertTrue(plan instanceof SplitNormalizationPlan); + } + + // test when target region size is 200 + when(masterServices.getTableDescriptors().get(any()).getNormalizerTargetRegionSize()) + .thenReturn(200L); + plans = normalizer.computePlanForTable(tableName); + Assert.assertEquals(2, plans.size()); + NormalizationPlan plan = plans.get(0); + assertTrue(plan instanceof MergeNormalizationPlan); + assertEquals(hri1, ((MergeNormalizationPlan) plan).getFirstRegion()); + assertEquals(hri2, ((MergeNormalizationPlan) plan).getSecondRegion()); + } + + @Test + public void testSplitWithTargetRegionSize() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + List RegionInfo = new ArrayList<>(); + Map regionSizes = new HashMap<>(); + + RegionInfo hri1 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("aaa")) + .setEndKey(Bytes.toBytes("bbb")).build(); + RegionInfo.add(hri1); + regionSizes.put(hri1.getRegionName(), 20); + + RegionInfo hri2 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("bbb")) + .setEndKey(Bytes.toBytes("ccc")).build(); + RegionInfo.add(hri2); + regionSizes.put(hri2.getRegionName(), 40); + + RegionInfo hri3 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("ccc")) + .setEndKey(Bytes.toBytes("ddd")).build(); + RegionInfo.add(hri3); + regionSizes.put(hri3.getRegionName(), 60); + + RegionInfo hri4 = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("ddd")) + .setEndKey(Bytes.toBytes("eee")).build(); + RegionInfo.add(hri4); + regionSizes.put(hri4.getRegionName(), 80); + + setupMocksForNormalizer(regionSizes, RegionInfo); + + // test when target region count is 8 + when(masterServices.getTableDescriptors().get(any()).getNormalizerTargetRegionCount()) + .thenReturn(8); + List plans = normalizer.computePlanForTable(tableName); + Assert.assertEquals(2, plans.size()); + + for (NormalizationPlan plan : plans) { + assertTrue(plan instanceof SplitNormalizationPlan); + } + + // test when target region count is 3 + when(masterServices.getTableDescriptors().get(any()).getNormalizerTargetRegionCount()) + .thenReturn(3); + plans = normalizer.computePlanForTable(tableName); + Assert.assertEquals(1, plans.size()); + NormalizationPlan plan = plans.get(0); + assertTrue(plan instanceof MergeNormalizationPlan); + assertEquals(hri1, ((MergeNormalizationPlan) plan).getFirstRegion()); + assertEquals(hri2, ((MergeNormalizationPlan) plan).getSecondRegion()); + } + @SuppressWarnings("MockitoCast") protected void setupMocksForNormalizer(Map regionSizes, List RegionInfo) { diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index c385d3dabf6..4385acf8b14 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -1237,6 +1237,8 @@ module Hbase htd.setReadOnly(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::READONLY))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::READONLY) htd.setCompactionEnabled(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::COMPACTION_ENABLED))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::COMPACTION_ENABLED) htd.setNormalizationEnabled(JBoolean.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::NORMALIZATION_ENABLED))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::NORMALIZATION_ENABLED) + htd.setNormalizerTargetRegionCount(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::NORMALIZER_TARGET_REGION_COUNT))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::NORMALIZER_TARGET_REGION_COUNT) + htd.setNormalizerTargetRegionSize(JLong.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::NORMALIZER_TARGET_REGION_SIZE))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::NORMALIZER_TARGET_REGION_SIZE) htd.setMemStoreFlushSize(JLong.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::MEMSTORE_FLUSHSIZE))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::MEMSTORE_FLUSHSIZE) htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::DURABILITY))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::DURABILITY) htd.setPriority(JInteger.valueOf(arg.delete(org.apache.hadoop.hbase.HTableDescriptor::PRIORITY))) if arg.include?(org.apache.hadoop.hbase.HTableDescriptor::PRIORITY) diff --git a/hbase-shell/src/main/ruby/shell/commands/alter.rb b/hbase-shell/src/main/ruby/shell/commands/alter.rb index 4aef28c1fc6..5ba755120f9 100644 --- a/hbase-shell/src/main/ruby/shell/commands/alter.rb +++ b/hbase-shell/src/main/ruby/shell/commands/alter.rb @@ -47,7 +47,8 @@ To delete the 'f1' column family in table 'ns1:t1', use one of: hbase> alter 'ns1:t1', 'delete' => 'f1' You can also change table-scope attributes like MAX_FILESIZE, READONLY, -MEMSTORE_FLUSHSIZE, DURABILITY, etc. These can be put at the end; +MEMSTORE_FLUSHSIZE, NORMALIZATION_ENABLED, NORMALIZER_TARGET_REGION_COUNT, +NORMALIZER_TARGET_REGION_SIZE(MB), DURABILITY, etc. These can be put at the end; for example, to change the max size of a region to 128MB, do: hbase> alter 't1', MAX_FILESIZE => '134217728' diff --git a/hbase-shell/src/test/ruby/hbase/admin_test.rb b/hbase-shell/src/test/ruby/hbase/admin_test.rb index 69dde45f5e8..1a129217a65 100644 --- a/hbase-shell/src/test/ruby/hbase/admin_test.rb +++ b/hbase-shell/src/test/ruby/hbase/admin_test.rb @@ -516,6 +516,13 @@ module Hbase assert_match(/12345678/, admin.describe(@test_name)) end + define_test 'alter should be able to set the TargetRegionSize and TargetRegionCount' do + command(:alter, @test_name, 'NORMALIZER_TARGET_REGION_COUNT' => 156) + assert_match(/156/, admin.describe(@test_name)) + command(:alter, @test_name, 'NORMALIZER_TARGET_REGION_SIZE' => 234) + assert_match(/234/, admin.describe(@test_name)) + end + define_test 'alter should support shortcut DELETE alter specs' do assert_equal(['x:', 'y:'], table(@test_name).get_all_columns.sort) command(:alter, @test_name, 'delete' => 'y')