From 69ea6928bcd3cf0f564feec16d8134f3cee8541f Mon Sep 17 00:00:00 2001 From: zhangduo Date: Wed, 20 Dec 2017 21:59:41 +0800 Subject: [PATCH] HBASE-19559 Fix TestLogRolling.testLogRollOnDatanodeDeath --- .../apache/hadoop/hbase/wal/WALFactory.java | 2 +- .../wal/AbstractTestLogRolling.java | 18 ++++++++++-------- .../regionserver/wal/TestAsyncLogRolling.java | 1 + .../hbase/regionserver/wal/TestLogRolling.java | 14 ++++++++------ 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java index 0628f8652f8..d1068d1bea6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java @@ -87,7 +87,7 @@ public class WALFactory implements WALFileLengthProvider { public static final String WAL_PROVIDER = "hbase.wal.provider"; static final String DEFAULT_WAL_PROVIDER = Providers.defaultProvider.name(); - static final String META_WAL_PROVIDER = "hbase.wal.meta_provider"; + public static final String META_WAL_PROVIDER = "hbase.wal.meta_provider"; static final String DEFAULT_META_WAL_PROVIDER = Providers.defaultProvider.name(); final String factoryId; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java index a323db6ac47..264b4c82195 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java @@ -26,21 +26,21 @@ import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; -import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; @@ -89,11 +89,9 @@ public abstract class AbstractTestLogRolling { // to the HDFS & HBase cluster startup. @BeforeClass public static void setUpBeforeClass() throws Exception { - - /**** configuration for testLogRolling ****/ // Force a region split after every 768KB - Configuration conf= TEST_UTIL.getConfiguration(); + Configuration conf = TEST_UTIL.getConfiguration(); conf.setLong(HConstants.HREGION_MAX_FILESIZE, 768L * 1024L); // We roll the log after every 32 writes @@ -114,6 +112,10 @@ public abstract class AbstractTestLogRolling { // Reduce thread wake frequency so that other threads can get // a chance to run. conf.setInt(HConstants.THREAD_WAKE_FREQUENCY, 2 * 1000); + + // disable low replication check for log roller to get a more stable result + // TestWALOpenAfterDNRollingStart will test this option. + conf.setLong("hbase.regionserver.hlog.check.lowreplication.interval", 24L * 60 * 60 * 1000); } @Before @@ -305,8 +307,8 @@ public abstract class AbstractTestLogRolling { protected Table createTestTable(String tableName) throws IOException { // Create the test table and open it - HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); - desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(getName())) + .addColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build(); admin.createTable(desc); return TEST_UTIL.getConnection().getTable(desc.getTableName()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncLogRolling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncLogRolling.java index 73f70a79755..325da942b2a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncLogRolling.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncLogRolling.java @@ -47,6 +47,7 @@ public class TestAsyncLogRolling extends AbstractTestLogRolling { Configuration conf = TestAsyncLogRolling.TEST_UTIL.getConfiguration(); conf.setInt(AsyncFSWAL.ASYNC_WAL_CREATE_MAX_RETRIES, 100); conf.set(WALFactory.WAL_PROVIDER, "asyncfs"); + conf.set(WALFactory.META_WAL_PROVIDER, "asyncfs"); AbstractTestLogRolling.setUpBeforeClass(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java index 12278eb3c05..53b15714d31 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java @@ -34,16 +34,17 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CategoryBasedTimeout; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -89,6 +90,7 @@ public class TestLogRolling extends AbstractTestLogRolling { conf.setInt("hbase.regionserver.hlog.tolerable.lowreplication", 2); conf.setInt("hbase.regionserver.hlog.lowreplication.rolllimit", 3); conf.set(WALFactory.WAL_PROVIDER, "filesystem"); + conf.set(WALFactory.META_WAL_PROVIDER, "filesystem"); AbstractTestLogRolling.setUpBeforeClass(); } @@ -133,8 +135,8 @@ public class TestLogRolling extends AbstractTestLogRolling { this.server = cluster.getRegionServer(0); // Create the test table and open it - HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(getName())); - desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(getName())) + .addColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build(); admin.createTable(desc); Table table = TEST_UTIL.getConnection().getTable(desc.getTableName()); @@ -241,8 +243,8 @@ public class TestLogRolling extends AbstractTestLogRolling { this.server = cluster.getRegionServer(0); // Create the test table and open it - HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(getName())); - desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(getName())) + .addColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build(); admin.createTable(desc); Table table = TEST_UTIL.getConnection().getTable(desc.getTableName());