From ffb8c641d9acaa53cdfc66f36a6217f1f5526e11 Mon Sep 17 00:00:00 2001 From: zhangduo Date: Mon, 13 Nov 2017 22:03:48 +0800 Subject: [PATCH] HBASE-19243 Start mini cluster once before class for TestFIFOCompactionPolicy --- .../compactions/TestFIFOCompactionPolicy.java | 138 +++++++----------- 1 file changed, 50 insertions(+), 88 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestFIFOCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestFIFOCompactionPolicy.java index 166fede2d9e..d06855bfa7f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestFIFOCompactionPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestFIFOCompactionPolicy.java @@ -18,17 +18,18 @@ package org.apache.hadoop.hbase.regionserver.compactions; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.List; -import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter.ExplainingPredicate; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Put; @@ -40,7 +41,6 @@ import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.HStore; -import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.Bytes; @@ -49,30 +49,33 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.TimeOffsetEnvironmentEdge; import org.junit.AfterClass; -import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.ExpectedException; @Category({ RegionServerTests.class, MediumTests.class }) public class TestFIFOCompactionPolicy { private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private final TableName tableName = TableName.valueOf(getClass().getSimpleName()); private final byte[] family = Bytes.toBytes("f"); private final byte[] qualifier = Bytes.toBytes("q"); + @Rule + public ExpectedException error = ExpectedException.none(); + private HStore getStoreWithName(TableName tableName) { MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster(); List rsts = cluster.getRegionServerThreads(); for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) { HRegionServer hrs = rsts.get(i).getRegionServer(); - for (Region region : hrs.getRegions(tableName)) { - return ((HRegion) region).getStores().iterator().next(); + for (HRegion region : hrs.getRegions(tableName)) { + return region.getStores().iterator().next(); } } return null; @@ -80,10 +83,6 @@ public class TestFIFOCompactionPolicy { private HStore prepareData() throws IOException { Admin admin = TEST_UTIL.getAdmin(); - if (admin.tableExists(tableName)) { - admin.disableTable(tableName); - admin.deleteTable(tableName); - } TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()) @@ -91,16 +90,14 @@ public class TestFIFOCompactionPolicy { DisabledRegionSplitPolicy.class.getName()) .addColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1).build()) .build(); - admin.createTable(desc); Table table = TEST_UTIL.getConnection().getTable(tableName); - Random rand = new Random(); TimeOffsetEnvironmentEdge edge = (TimeOffsetEnvironmentEdge) EnvironmentEdgeManager.getDelegate(); for (int i = 0; i < 10; i++) { for (int j = 0; j < 10; j++) { byte[] value = new byte[128 * 1024]; - rand.nextBytes(value); + ThreadLocalRandom.current().nextBytes(value); table.put(new Put(Bytes.toBytes(i * 10 + j)).addColumn(family, qualifier, value)); } admin.flush(tableName); @@ -109,77 +106,59 @@ public class TestFIFOCompactionPolicy { return getStoreWithName(tableName); } - @BeforeClass - public static void setEnvironmentEdge() - { + @BeforeClass + public static void setEnvironmentEdge() throws Exception { EnvironmentEdge ee = new TimeOffsetEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(ee); + Configuration conf = TEST_UTIL.getConfiguration(); + conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000); + TEST_UTIL.startMiniCluster(1); } - + @AfterClass - public static void resetEnvironmentEdge() - { + public static void resetEnvironmentEdge() throws Exception { + TEST_UTIL.shutdownMiniCluster(); EnvironmentEdgeManager.reset(); } - + @Test public void testPurgeExpiredFiles() throws Exception { - Configuration conf = TEST_UTIL.getConfiguration(); - conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000); + HStore store = prepareData(); + assertEquals(10, store.getStorefilesCount()); + TEST_UTIL.getAdmin().majorCompact(tableName); + TEST_UTIL.waitFor(30000, new ExplainingPredicate() { - TEST_UTIL.startMiniCluster(1); - try { - HStore store = prepareData(); - assertEquals(10, store.getStorefilesCount()); - TEST_UTIL.getAdmin().majorCompact(tableName); - while (store.getStorefilesCount() > 1) { - Thread.sleep(100); + @Override + public boolean evaluate() throws Exception { + return store.getStorefilesCount() == 1; } - assertTrue(store.getStorefilesCount() == 1); - } finally { - TEST_UTIL.shutdownMiniCluster(); - } - } - - @Test - public void testSanityCheckTTL() throws Exception { - Configuration conf = TEST_UTIL.getConfiguration(); - conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000); - TEST_UTIL.startMiniCluster(1); - Admin admin = TEST_UTIL.getAdmin(); + @Override + public String explainFailure() throws Exception { + return "The store file count " + store.getStorefilesCount() + " is still greater than 1"; + } + }); + } + + @Test + public void testSanityCheckTTL() throws IOException { + error.expect(DoNotRetryIOException.class); + error.expectMessage("Default TTL is not supported"); TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-TTL"); - if (admin.tableExists(tableName)) { - admin.disableTable(tableName); - admin.deleteTable(tableName); - } TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()) .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName()) .addColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build(); - try { - admin.createTable(desc); - Assert.fail(); - } catch (Exception e) { - } finally { - TEST_UTIL.shutdownMiniCluster(); - } + TEST_UTIL.getAdmin().createTable(desc); } @Test - public void testSanityCheckMinVersion() throws Exception { - Configuration conf = TEST_UTIL.getConfiguration(); - conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000); - TEST_UTIL.startMiniCluster(1); - - Admin admin = TEST_UTIL.getAdmin(); + public void testSanityCheckMinVersion() throws IOException { + error.expect(DoNotRetryIOException.class); + error.expectMessage("MIN_VERSION > 0 is not supported for FIFO compaction"); TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-MinVersion"); - if (admin.tableExists(tableName)) { - admin.disableTable(tableName); - admin.deleteTable(tableName); - } TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()) @@ -188,40 +167,23 @@ public class TestFIFOCompactionPolicy { .addColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1) .setMinVersions(1).build()) .build(); - try { - admin.createTable(desc); - Assert.fail(); - } catch (Exception e) { - } finally { - TEST_UTIL.shutdownMiniCluster(); - } + TEST_UTIL.getAdmin().createTable(desc); } - - @Test - public void testSanityCheckBlockingStoreFiles() throws Exception { - Configuration conf = TEST_UTIL.getConfiguration(); - conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10); - TEST_UTIL.startMiniCluster(1); - Admin admin = TEST_UTIL.getAdmin(); + @Test + public void testSanityCheckBlockingStoreFiles() throws IOException { + error.expect(DoNotRetryIOException.class); + error.expectMessage("Blocking file count 'hbase.hstore.blockingStoreFiles'"); + error.expectMessage("is below recommended minimum of 1000 for column family"); TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-BlockingStoreFiles"); - if (admin.tableExists(tableName)) { - admin.disableTable(tableName); - admin.deleteTable(tableName); - } TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()) .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName()) + .setValue(HStore.BLOCKING_STOREFILES_KEY, "10") .addColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1).build()) .build(); - try { - admin.createTable(desc); - Assert.fail(); - } catch (Exception e) { - } finally { - TEST_UTIL.shutdownMiniCluster(); - } + TEST_UTIL.getAdmin().createTable(desc); } }