Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-26827 RegionServer JVM crash when compact mob table

This commit is contained in:
meiyi 2022-03-11 16:44:46 +08:00 committed by meiyi
parent 816e919e95
commit 15e6f2bd74
3 changed files with 31 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
View File

@ -550,12 +550,12 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
progress.cancel(); progress.cancel();
return false; return false;
} }
}
if (kvs != null && bytesWrittenProgressForShippedCall > shippedCallSizeLimit) { if (kvs != null && bytesWrittenProgressForShippedCall > shippedCallSizeLimit) {
((ShipperListener) writer).beforeShipped(); ((ShipperListener) writer).beforeShipped();
kvs.shipped(); kvs.shipped();
bytesWrittenProgressForShippedCall = 0; bytesWrittenProgressForShippedCall = 0;
} }
}
// Log the progress of long running compactions every minute if // Log the progress of long running compactions every minute if
// logging at DEBUG level // logging at DEBUG level
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mob/FaultyMobStoreCompactor.java
View File

@ -323,12 +323,12 @@ public class FaultyMobStoreCompactor extends DefaultMobStoreCompactor {
progress.cancel(); progress.cancel();
return false; return false;
} }
}
if (kvs != null && bytesWrittenProgressForShippedCall > shippedCallSizeLimit) { if (kvs != null && bytesWrittenProgressForShippedCall > shippedCallSizeLimit) {
((ShipperListener) writer).beforeShipped(); ((ShipperListener) writer).beforeShipped();
kvs.shipped(); kvs.shipped();
bytesWrittenProgressForShippedCall = 0; bytesWrittenProgressForShippedCall = 0;
} }
}
// Log the progress of long running compactions every minute if // Log the progress of long running compactions every minute if
// logging at DEBUG level // logging at DEBUG level
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {

28
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionWithByteBuff.java
View File

@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.io.ByteBuffAllocator; import org.apache.hadoop.hbase.io.ByteBuffAllocator;
import org.apache.hadoop.hbase.io.DeallocateRewriteByteBuffAllocator;
import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.JVMClusterUtil;
@ -64,6 +65,8 @@ public class TestCompactionWithByteBuff {
@BeforeClass @BeforeClass
public static void setupBeforeClass() throws Exception { public static void setupBeforeClass() throws Exception {
conf.setBoolean(ByteBuffAllocator.ALLOCATOR_POOL_ENABLED_KEY, true); conf.setBoolean(ByteBuffAllocator.ALLOCATOR_POOL_ENABLED_KEY, true);
conf.set(ByteBuffAllocator.BYTEBUFF_ALLOCATOR_CLASS,
DeallocateRewriteByteBuffAllocator.class.getName());
conf.setInt(ByteBuffAllocator.BUFFER_SIZE_KEY, 1024 * 5); conf.setInt(ByteBuffAllocator.BUFFER_SIZE_KEY, 1024 * 5);
conf.setInt(CompactSplit.SMALL_COMPACTION_THREADS, REGION_COUNT * 2); conf.setInt(CompactSplit.SMALL_COMPACTION_THREADS, REGION_COUNT * 2);
conf.setInt(CompactSplit.LARGE_COMPACTION_THREADS, REGION_COUNT * 2); conf.setInt(CompactSplit.LARGE_COMPACTION_THREADS, REGION_COUNT * 2);
@ -78,11 +81,9 @@ public class TestCompactionWithByteBuff {
TEST_UTIL.shutdownMiniCluster(); TEST_UTIL.shutdownMiniCluster();
} }
@Test private void testCompaction(TableName table, boolean isMob) throws Exception {
public void testCompaction() throws Exception {
TableName table = TableName.valueOf("t1");
admin.compactionSwitch(false, new ArrayList<>(0)); admin.compactionSwitch(false, new ArrayList<>(0));
try (Table t = createTable(TEST_UTIL, table)) { try (Table t = createTable(TEST_UTIL, table, isMob)) {
for (int i = 0; i < 2; i++) { for (int i = 0; i < 2; i++) {
put(t); put(t);
admin.flush(table); admin.flush(table);
@ -108,9 +109,22 @@ public class TestCompactionWithByteBuff {
} }
} }
private Table createTable(HBaseTestingUtil util, TableName tableName) throws IOException { @Test
TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily( public void testCompaction() throws Exception {
ColumnFamilyDescriptorBuilder.newBuilder(COLUMN).setBlocksize(1024 * 4).build()).build(); testCompaction(TableName.valueOf(name.getMethodName()), false);
}
@Test
public void testCompactionForMobTable() throws Exception {
testCompaction(TableName.valueOf(name.getMethodName()), true);
}
private Table createTable(HBaseTestingUtil util, TableName tableName, boolean isMob)
throws IOException {
TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName)
.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(COLUMN).setBlocksize(1024 * 4)
.setMobEnabled(isMob).setMobThreshold(10240).build())
.build();
byte[][] splits = new byte[REGION_COUNT - 1][]; byte[][] splits = new byte[REGION_COUNT - 1][];
for (int i = 1; i < REGION_COUNT; i++) { for (int i = 1; i < REGION_COUNT; i++) {
splits[i - 1] = Bytes.toBytes(buildRow((int) (ROW_COUNT / REGION_COUNT * i))); splits[i - 1] = Bytes.toBytes(buildRow((int) (ROW_COUNT / REGION_COUNT * i)));