Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-19559 Fix TestLogRolling.testLogRollOnDatanodeDeath

This commit is contained in:
zhangduo 2017-12-20 21:59:41 +08:00
parent 992b5d8630
commit 69ea6928bc
4 changed files with 20 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
View File

@ -87,7 +87,7 @@ public class WALFactory implements WALFileLengthProvider {
public static final String WAL_PROVIDER = "hbase.wal.provider"; public static final String WAL_PROVIDER = "hbase.wal.provider";
static final String DEFAULT_WAL_PROVIDER = Providers.defaultProvider.name(); static final String DEFAULT_WAL_PROVIDER = Providers.defaultProvider.name();
static final String META_WAL_PROVIDER = "hbase.wal.meta_provider"; public static final String META_WAL_PROVIDER = "hbase.wal.meta_provider";
static final String DEFAULT_META_WAL_PROVIDER = Providers.defaultProvider.name(); static final String DEFAULT_META_WAL_PROVIDER = Providers.defaultProvider.name();
final String factoryId; final String factoryId;

18
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestLogRolling.java
View File

@ -26,21 +26,21 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.Threads;
@ -89,11 +89,9 @@ public abstract class AbstractTestLogRolling {
// to the HDFS & HBase cluster startup. // to the HDFS & HBase cluster startup.
@BeforeClass @BeforeClass
public static void setUpBeforeClass() throws Exception { public static void setUpBeforeClass() throws Exception {
/**** configuration for testLogRolling ****/ /**** configuration for testLogRolling ****/
// Force a region split after every 768KB // Force a region split after every 768KB
Configuration conf= TEST_UTIL.getConfiguration(); Configuration conf = TEST_UTIL.getConfiguration();
conf.setLong(HConstants.HREGION_MAX_FILESIZE, 768L * 1024L); conf.setLong(HConstants.HREGION_MAX_FILESIZE, 768L * 1024L);
// We roll the log after every 32 writes // We roll the log after every 32 writes
@ -114,6 +112,10 @@ public abstract class AbstractTestLogRolling {
// Reduce thread wake frequency so that other threads can get // Reduce thread wake frequency so that other threads can get
// a chance to run. // a chance to run.
conf.setInt(HConstants.THREAD_WAKE_FREQUENCY, 2 * 1000); conf.setInt(HConstants.THREAD_WAKE_FREQUENCY, 2 * 1000);
// disable low replication check for log roller to get a more stable result
// TestWALOpenAfterDNRollingStart will test this option.
conf.setLong("hbase.regionserver.hlog.check.lowreplication.interval", 24L * 60 * 60 * 1000);
} }
@Before @Before
@ -305,8 +307,8 @@ public abstract class AbstractTestLogRolling {
protected Table createTestTable(String tableName) throws IOException { protected Table createTestTable(String tableName) throws IOException {
// Create the test table and open it // Create the test table and open it
HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(getName()))
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); .addColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
admin.createTable(desc); admin.createTable(desc);
return TEST_UTIL.getConnection().getTable(desc.getTableName()); return TEST_UTIL.getConnection().getTable(desc.getTableName());
} }

1
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestAsyncLogRolling.java
View File

@ -47,6 +47,7 @@ public class TestAsyncLogRolling extends AbstractTestLogRolling {
Configuration conf = TestAsyncLogRolling.TEST_UTIL.getConfiguration(); Configuration conf = TestAsyncLogRolling.TEST_UTIL.getConfiguration();
conf.setInt(AsyncFSWAL.ASYNC_WAL_CREATE_MAX_RETRIES, 100); conf.setInt(AsyncFSWAL.ASYNC_WAL_CREATE_MAX_RETRIES, 100);
conf.set(WALFactory.WAL_PROVIDER, "asyncfs"); conf.set(WALFactory.WAL_PROVIDER, "asyncfs");
conf.set(WALFactory.META_WAL_PROVIDER, "asyncfs");
AbstractTestLogRolling.setUpBeforeClass(); AbstractTestLogRolling.setUpBeforeClass();
} }

14
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
View File

@ -34,16 +34,17 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.CategoryBasedTimeout; import org.apache.hadoop.hbase.CategoryBasedTimeout;
import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.LargeTests;
@ -89,6 +90,7 @@ public class TestLogRolling extends AbstractTestLogRolling {
conf.setInt("hbase.regionserver.hlog.tolerable.lowreplication", 2); conf.setInt("hbase.regionserver.hlog.tolerable.lowreplication", 2);
conf.setInt("hbase.regionserver.hlog.lowreplication.rolllimit", 3); conf.setInt("hbase.regionserver.hlog.lowreplication.rolllimit", 3);
conf.set(WALFactory.WAL_PROVIDER, "filesystem"); conf.set(WALFactory.WAL_PROVIDER, "filesystem");
conf.set(WALFactory.META_WAL_PROVIDER, "filesystem");
AbstractTestLogRolling.setUpBeforeClass(); AbstractTestLogRolling.setUpBeforeClass();
} }
@ -133,8 +135,8 @@ public class TestLogRolling extends AbstractTestLogRolling {
this.server = cluster.getRegionServer(0); this.server = cluster.getRegionServer(0);
// Create the test table and open it // Create the test table and open it
HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(getName())); TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(getName()))
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); .addColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
admin.createTable(desc); admin.createTable(desc);
Table table = TEST_UTIL.getConnection().getTable(desc.getTableName()); Table table = TEST_UTIL.getConnection().getTable(desc.getTableName());
@ -241,8 +243,8 @@ public class TestLogRolling extends AbstractTestLogRolling {
this.server = cluster.getRegionServer(0); this.server = cluster.getRegionServer(0);
// Create the test table and open it // Create the test table and open it
HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(getName())); TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(getName()))
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); .addColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
admin.createTable(desc); admin.createTable(desc);
Table table = TEST_UTIL.getConnection().getTable(desc.getTableName()); Table table = TEST_UTIL.getConnection().getTable(desc.getTableName());