HBASE-21141 Enable MOB in backup / restore test involving incremental backup
Signed-off-by: tedyu <yuzhihong@gmail.com>
This commit is contained in:
Artem Ervits
tedyu
parent
43a10df70f
commit
825e14b68e
|
|
@ -63,7 +63,7 @@ public class TestIncrementalBackup extends TestBackupBase {
|
||||||
@Parameterized.Parameters
|
@Parameterized.Parameters
|
||||||
public static Collection<Object[]> data() {
|
public static Collection<Object[]> data() {
|
||||||
provider = "multiwal";
|
provider = "multiwal";
|
||||||
List<Object[]> params = new ArrayList<Object[]>();
|
List<Object[]> params = new ArrayList<>();
|
||||||
params.add(new Object[] { Boolean.TRUE });
|
params.add(new Object[] { Boolean.TRUE });
|
||||||
return params;
|
return params;
|
||||||
}
|
}
|
||||||
|
|
@ -71,146 +71,157 @@ public class TestIncrementalBackup extends TestBackupBase {
|
||||||
public TestIncrementalBackup(Boolean b) {
|
public TestIncrementalBackup(Boolean b) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// implement all test cases in 1 test since incremental backup/restore has dependencies
|
// implement all test cases in 1 test since incremental
|
||||||
|
// backup/restore has dependencies
|
||||||
@Test
|
@Test
|
||||||
public void TestIncBackupRestore() throws Exception {
|
public void TestIncBackupRestore() throws Exception {
|
||||||
|
|
||||||
int ADD_ROWS = 99;
|
int ADD_ROWS = 99;
|
||||||
|
|
||||||
// #1 - create full backup for all tables
|
// #1 - create full backup for all tables
|
||||||
LOG.info("create full backup image for all tables");
|
LOG.info("create full backup image for all tables");
|
||||||
|
|
||||||
List<TableName> tables = Lists.newArrayList(table1, table2);
|
List<TableName> tables = Lists.newArrayList(table1, table2);
|
||||||
final byte[] fam3Name = Bytes.toBytes("f3");
|
final byte[] fam3Name = Bytes.toBytes("f3");
|
||||||
|
final byte[] mobName = Bytes.toBytes("mob");
|
||||||
|
|
||||||
table1Desc.addFamily(new HColumnDescriptor(fam3Name));
|
table1Desc.addFamily(new HColumnDescriptor(fam3Name));
|
||||||
|
HColumnDescriptor mobHcd = new HColumnDescriptor(mobName);
|
||||||
|
mobHcd.setMobEnabled(true);
|
||||||
|
mobHcd.setMobThreshold(5L);
|
||||||
|
table1Desc.addFamily(mobHcd);
|
||||||
HBaseTestingUtility.modifyTableSync(TEST_UTIL.getAdmin(), table1Desc);
|
HBaseTestingUtility.modifyTableSync(TEST_UTIL.getAdmin(), table1Desc);
|
||||||
|
|
||||||
Connection conn = ConnectionFactory.createConnection(conf1);
|
try (Connection conn = ConnectionFactory.createConnection(conf1)) {
|
||||||
int NB_ROWS_FAM3 = 6;
|
int NB_ROWS_FAM3 = 6;
|
||||||
insertIntoTable(conn, table1, fam3Name, 3, NB_ROWS_FAM3).close();
|
insertIntoTable(conn, table1, fam3Name, 3, NB_ROWS_FAM3).close();
|
||||||
|
insertIntoTable(conn, table1, mobName, 3, NB_ROWS_FAM3).close();
|
||||||
|
HBaseAdmin admin = null;
|
||||||
|
admin = (HBaseAdmin) conn.getAdmin();
|
||||||
|
BackupAdminImpl client = new BackupAdminImpl(conn);
|
||||||
|
BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR);
|
||||||
|
String backupIdFull = client.backupTables(request);
|
||||||
|
assertTrue(checkSucceeded(backupIdFull));
|
||||||
|
|
||||||
HBaseAdmin admin = null;
|
// #2 - insert some data to table
|
||||||
admin = (HBaseAdmin) conn.getAdmin();
|
HTable t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS);
|
||||||
BackupAdminImpl client = new BackupAdminImpl(conn);
|
LOG.debug("writing " + ADD_ROWS + " rows to " + table1);
|
||||||
|
Assert.assertEquals(HBaseTestingUtility.countRows(t1),
|
||||||
|
NB_ROWS_IN_BATCH + ADD_ROWS + NB_ROWS_FAM3);
|
||||||
|
LOG.debug("written " + ADD_ROWS + " rows to " + table1);
|
||||||
|
|
||||||
BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR);
|
// additionally, insert rows to MOB cf
|
||||||
String backupIdFull = client.backupTables(request);
|
int NB_ROWS_MOB = 111;
|
||||||
|
insertIntoTable(conn, table1, mobName, 3, NB_ROWS_MOB);
|
||||||
|
LOG.debug("written " + NB_ROWS_MOB + " rows to " + table1 + " to Mob enabled CF");
|
||||||
|
t1.close();
|
||||||
|
Assert.assertEquals(HBaseTestingUtility.countRows(t1),
|
||||||
|
NB_ROWS_IN_BATCH + ADD_ROWS + NB_ROWS_MOB);
|
||||||
|
|
||||||
assertTrue(checkSucceeded(backupIdFull));
|
HTable t2 = (HTable) conn.getTable(table2);
|
||||||
|
Put p2;
|
||||||
|
for (int i = 0; i < 5; i++) {
|
||||||
|
p2 = new Put(Bytes.toBytes("row-t2" + i));
|
||||||
|
p2.addColumn(famName, qualName, Bytes.toBytes("val" + i));
|
||||||
|
t2.put(p2);
|
||||||
|
}
|
||||||
|
Assert.assertEquals(NB_ROWS_IN_BATCH + 5, HBaseTestingUtility.countRows(t2));
|
||||||
|
t2.close();
|
||||||
|
LOG.debug("written " + 5 + " rows to " + table2);
|
||||||
|
|
||||||
// #2 - insert some data to table
|
// split table1
|
||||||
HTable t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS);
|
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
|
||||||
LOG.debug("writing " + ADD_ROWS + " rows to " + table1);
|
List<HRegion> regions = cluster.getRegions(table1);
|
||||||
|
byte[] name = regions.get(0).getRegionInfo().getRegionName();
|
||||||
|
long startSplitTime = EnvironmentEdgeManager.currentTime();
|
||||||
|
|
||||||
Assert.assertEquals(TEST_UTIL.countRows(t1), NB_ROWS_IN_BATCH + ADD_ROWS + NB_ROWS_FAM3);
|
try {
|
||||||
t1.close();
|
admin.splitRegion(name);
|
||||||
LOG.debug("written " + ADD_ROWS + " rows to " + table1);
|
} catch (IOException e) {
|
||||||
|
// although split fail, this may not affect following check in current API,
|
||||||
|
// exception will be thrown.
|
||||||
|
LOG.debug("region is not splittable, because " + e);
|
||||||
|
}
|
||||||
|
while (!admin.isTableAvailable(table1)) {
|
||||||
|
Thread.sleep(100);
|
||||||
|
}
|
||||||
|
|
||||||
HTable t2 = (HTable) conn.getTable(table2);
|
long endSplitTime = EnvironmentEdgeManager.currentTime();
|
||||||
Put p2;
|
// split finished
|
||||||
for (int i = 0; i < 5; i++) {
|
LOG.debug("split finished in =" + (endSplitTime - startSplitTime));
|
||||||
p2 = new Put(Bytes.toBytes("row-t2" + i));
|
|
||||||
p2.addColumn(famName, qualName, Bytes.toBytes("val" + i));
|
// #3 - incremental backup for multiple tables
|
||||||
t2.put(p2);
|
tables = Lists.newArrayList(table1, table2);
|
||||||
|
request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
|
||||||
|
String backupIdIncMultiple = client.backupTables(request);
|
||||||
|
assertTrue(checkSucceeded(backupIdIncMultiple));
|
||||||
|
|
||||||
|
// add column family f2 to table1
|
||||||
|
final byte[] fam2Name = Bytes.toBytes("f2");
|
||||||
|
table1Desc.addFamily(new HColumnDescriptor(fam2Name));
|
||||||
|
|
||||||
|
// drop column family f3
|
||||||
|
table1Desc.removeFamily(fam3Name);
|
||||||
|
HBaseTestingUtility.modifyTableSync(TEST_UTIL.getAdmin(), table1Desc);
|
||||||
|
|
||||||
|
int NB_ROWS_FAM2 = 7;
|
||||||
|
HTable t3 = insertIntoTable(conn, table1, fam2Name, 2, NB_ROWS_FAM2);
|
||||||
|
t3.close();
|
||||||
|
|
||||||
|
// Wait for 5 sec to make sure that old WALs were deleted
|
||||||
|
Thread.sleep(5000);
|
||||||
|
|
||||||
|
// #4 - additional incremental backup for multiple tables
|
||||||
|
request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
|
||||||
|
String backupIdIncMultiple2 = client.backupTables(request);
|
||||||
|
assertTrue(checkSucceeded(backupIdIncMultiple2));
|
||||||
|
|
||||||
|
// #5 - restore full backup for all tables
|
||||||
|
TableName[] tablesRestoreFull = new TableName[] { table1, table2 };
|
||||||
|
TableName[] tablesMapFull = new TableName[] { table1_restore, table2_restore };
|
||||||
|
|
||||||
|
LOG.debug("Restoring full " + backupIdFull);
|
||||||
|
client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdFull, false,
|
||||||
|
tablesRestoreFull, tablesMapFull, true));
|
||||||
|
|
||||||
|
// #6.1 - check tables for full restore
|
||||||
|
HBaseAdmin hAdmin = TEST_UTIL.getHBaseAdmin();
|
||||||
|
assertTrue(hAdmin.tableExists(table1_restore));
|
||||||
|
assertTrue(hAdmin.tableExists(table2_restore));
|
||||||
|
hAdmin.close();
|
||||||
|
|
||||||
|
// #6.2 - checking row count of tables for full restore
|
||||||
|
HTable hTable = (HTable) conn.getTable(table1_restore);
|
||||||
|
Assert.assertEquals(HBaseTestingUtility.countRows(hTable), NB_ROWS_IN_BATCH + NB_ROWS_FAM3);
|
||||||
|
hTable.close();
|
||||||
|
|
||||||
|
hTable = (HTable) conn.getTable(table2_restore);
|
||||||
|
Assert.assertEquals(NB_ROWS_IN_BATCH, HBaseTestingUtility.countRows(hTable));
|
||||||
|
hTable.close();
|
||||||
|
|
||||||
|
// #7 - restore incremental backup for multiple tables, with overwrite
|
||||||
|
TableName[] tablesRestoreIncMultiple = new TableName[] { table1, table2 };
|
||||||
|
TableName[] tablesMapIncMultiple = new TableName[] { table1_restore, table2_restore };
|
||||||
|
client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple2,
|
||||||
|
false, tablesRestoreIncMultiple, tablesMapIncMultiple, true));
|
||||||
|
hTable = (HTable) conn.getTable(table1_restore);
|
||||||
|
|
||||||
|
LOG.debug("After incremental restore: " + hTable.getDescriptor());
|
||||||
|
int countFamName = TEST_UTIL.countRows(hTable, famName);
|
||||||
|
LOG.debug("f1 has " + countFamName + " rows");
|
||||||
|
Assert.assertEquals(countFamName, NB_ROWS_IN_BATCH + ADD_ROWS);
|
||||||
|
|
||||||
|
int countFam2Name = TEST_UTIL.countRows(hTable, fam2Name);
|
||||||
|
LOG.debug("f2 has " + countFam2Name + " rows");
|
||||||
|
Assert.assertEquals(countFam2Name, NB_ROWS_FAM2);
|
||||||
|
|
||||||
|
int countMobName = TEST_UTIL.countRows(hTable, mobName);
|
||||||
|
LOG.debug("mob has " + countMobName + " rows");
|
||||||
|
Assert.assertEquals(countMobName, NB_ROWS_MOB);
|
||||||
|
hTable.close();
|
||||||
|
|
||||||
|
hTable = (HTable) conn.getTable(table2_restore);
|
||||||
|
Assert.assertEquals(NB_ROWS_IN_BATCH + 5, HBaseTestingUtility.countRows(hTable));
|
||||||
|
hTable.close();
|
||||||
|
admin.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
Assert.assertEquals(TEST_UTIL.countRows(t2), NB_ROWS_IN_BATCH + 5);
|
|
||||||
t2.close();
|
|
||||||
LOG.debug("written " + 5 + " rows to " + table2);
|
|
||||||
// split table1
|
|
||||||
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
|
|
||||||
List<HRegion> regions = cluster.getRegions(table1);
|
|
||||||
|
|
||||||
byte[] name = regions.get(0).getRegionInfo().getRegionName();
|
|
||||||
long startSplitTime = EnvironmentEdgeManager.currentTime();
|
|
||||||
try {
|
|
||||||
admin.splitRegion(name);
|
|
||||||
} catch (IOException e) {
|
|
||||||
//although split fail, this may not affect following check
|
|
||||||
//In old split without AM2, if region's best split key is not found,
|
|
||||||
//there are not exception thrown. But in current API, exception
|
|
||||||
//will be thrown.
|
|
||||||
LOG.debug("region is not splittable, because " + e);
|
|
||||||
}
|
|
||||||
|
|
||||||
while (!admin.isTableAvailable(table1)) {
|
|
||||||
Thread.sleep(100);
|
|
||||||
}
|
|
||||||
|
|
||||||
long endSplitTime = EnvironmentEdgeManager.currentTime();
|
|
||||||
|
|
||||||
// split finished
|
|
||||||
LOG.debug("split finished in =" + (endSplitTime - startSplitTime));
|
|
||||||
|
|
||||||
// #3 - incremental backup for multiple tables
|
|
||||||
tables = Lists.newArrayList(table1, table2);
|
|
||||||
request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
|
|
||||||
String backupIdIncMultiple = client.backupTables(request);
|
|
||||||
assertTrue(checkSucceeded(backupIdIncMultiple));
|
|
||||||
|
|
||||||
// add column family f2 to table1
|
|
||||||
final byte[] fam2Name = Bytes.toBytes("f2");
|
|
||||||
table1Desc.addFamily(new HColumnDescriptor(fam2Name));
|
|
||||||
// drop column family f3
|
|
||||||
table1Desc.removeFamily(fam3Name);
|
|
||||||
HBaseTestingUtility.modifyTableSync(TEST_UTIL.getAdmin(), table1Desc);
|
|
||||||
|
|
||||||
int NB_ROWS_FAM2 = 7;
|
|
||||||
HTable t3 = insertIntoTable(conn, table1, fam2Name, 2, NB_ROWS_FAM2);
|
|
||||||
t3.close();
|
|
||||||
// Wait for 5 sec to make sure that old WALs were deleted
|
|
||||||
Thread.sleep(5000);
|
|
||||||
|
|
||||||
// #3 - incremental backup for multiple tables
|
|
||||||
request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
|
|
||||||
String backupIdIncMultiple2 = client.backupTables(request);
|
|
||||||
assertTrue(checkSucceeded(backupIdIncMultiple2));
|
|
||||||
|
|
||||||
// #4 - restore full backup for all tables
|
|
||||||
TableName[] tablesRestoreFull = new TableName[] { table1, table2 };
|
|
||||||
|
|
||||||
TableName[] tablesMapFull = new TableName[] { table1_restore, table2_restore };
|
|
||||||
|
|
||||||
LOG.debug("Restoring full " + backupIdFull);
|
|
||||||
client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdFull, false,
|
|
||||||
tablesRestoreFull, tablesMapFull, true));
|
|
||||||
|
|
||||||
// #5.1 - check tables for full restore
|
|
||||||
HBaseAdmin hAdmin = TEST_UTIL.getHBaseAdmin();
|
|
||||||
assertTrue(hAdmin.tableExists(table1_restore));
|
|
||||||
assertTrue(hAdmin.tableExists(table2_restore));
|
|
||||||
|
|
||||||
hAdmin.close();
|
|
||||||
|
|
||||||
// #5.2 - checking row count of tables for full restore
|
|
||||||
HTable hTable = (HTable) conn.getTable(table1_restore);
|
|
||||||
Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH + NB_ROWS_FAM3);
|
|
||||||
hTable.close();
|
|
||||||
|
|
||||||
hTable = (HTable) conn.getTable(table2_restore);
|
|
||||||
Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH);
|
|
||||||
hTable.close();
|
|
||||||
|
|
||||||
// #6 - restore incremental backup for multiple tables, with overwrite
|
|
||||||
TableName[] tablesRestoreIncMultiple = new TableName[] { table1, table2 };
|
|
||||||
TableName[] tablesMapIncMultiple = new TableName[] { table1_restore, table2_restore };
|
|
||||||
client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple2,
|
|
||||||
false, tablesRestoreIncMultiple, tablesMapIncMultiple, true));
|
|
||||||
|
|
||||||
hTable = (HTable) conn.getTable(table1_restore);
|
|
||||||
LOG.debug("After incremental restore: " + hTable.getDescriptor());
|
|
||||||
LOG.debug("f1 has " + TEST_UTIL.countRows(hTable, famName) + " rows");
|
|
||||||
Assert.assertEquals(TEST_UTIL.countRows(hTable, famName), NB_ROWS_IN_BATCH + ADD_ROWS);
|
|
||||||
LOG.debug("f2 has " + TEST_UTIL.countRows(hTable, fam2Name) + " rows");
|
|
||||||
Assert.assertEquals(TEST_UTIL.countRows(hTable, fam2Name), NB_ROWS_FAM2);
|
|
||||||
hTable.close();
|
|
||||||
|
|
||||||
hTable = (HTable) conn.getTable(table2_restore);
|
|
||||||
Assert.assertEquals(TEST_UTIL.countRows(hTable), NB_ROWS_IN_BATCH + 5);
|
|
||||||
hTable.close();
|
|
||||||
|
|
||||||
admin.close();
|
|
||||||
conn.close();
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue