HBASE-21194 Add tests in TestCopyTable which exercises MOB feature
Signed-off-by: tedyu <yuzhihong@gmail.com>
This commit is contained in:
Artem Ervits
tedyu
parent
7adf590106
commit
b723ce1051
|
|
@ -30,15 +30,20 @@ import org.apache.hadoop.hbase.CellUtil;
|
|||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.client.Get;
|
||||
import org.apache.hadoop.hbase.client.Put;
|
||||
import org.apache.hadoop.hbase.client.Result;
|
||||
import org.apache.hadoop.hbase.client.Table;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptor;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.mob.MobTestUtil;
|
||||
import org.apache.hadoop.hbase.testclassification.LargeTests;
|
||||
import org.apache.hadoop.hbase.testclassification.MapReduceTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.LauncherSecurityManager;
|
||||
import org.apache.hadoop.util.ToolRunner;
|
||||
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.ClassRule;
|
||||
|
|
@ -100,7 +105,7 @@ public class TestCopyTable {
|
|||
if (bulkload) {
|
||||
code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
|
||||
copy, new String[] { "--new.name=" + tableName2.getNameAsString(),
|
||||
"--bulkload", tableName1.getNameAsString() });
|
||||
"--bulkload", tableName1.getNameAsString() });
|
||||
} else {
|
||||
code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
|
||||
copy, new String[] { "--new.name=" + tableName2.getNameAsString(),
|
||||
|
|
@ -121,9 +126,74 @@ public class TestCopyTable {
|
|||
}
|
||||
}
|
||||
|
||||
private void doCopyTableTestWithMob(boolean bulkload) throws Exception {
|
||||
final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
|
||||
final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
|
||||
final byte[] FAMILY = Bytes.toBytes("mob");
|
||||
final byte[] COLUMN1 = Bytes.toBytes("c1");
|
||||
|
||||
ColumnFamilyDescriptorBuilder cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY);
|
||||
|
||||
cfd.setMobEnabled(true);
|
||||
cfd.setMobThreshold(5);
|
||||
TableDescriptor desc1 = TableDescriptorBuilder.newBuilder(tableName1)
|
||||
.setColumnFamily(cfd.build())
|
||||
.build();
|
||||
TableDescriptor desc2 = TableDescriptorBuilder.newBuilder(tableName2)
|
||||
.setColumnFamily(cfd.build())
|
||||
.build();
|
||||
|
||||
try (Table t1 = TEST_UTIL.createTable(desc1, null);
|
||||
Table t2 = TEST_UTIL.createTable(desc2, null);) {
|
||||
|
||||
// put rows into the first table
|
||||
for (int i = 0; i < 10; i++) {
|
||||
Put p = new Put(Bytes.toBytes("row" + i));
|
||||
p.addColumn(FAMILY, COLUMN1, COLUMN1);
|
||||
t1.put(p);
|
||||
}
|
||||
|
||||
CopyTable copy = new CopyTable();
|
||||
|
||||
int code;
|
||||
if (bulkload) {
|
||||
code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
|
||||
copy, new String[] { "--new.name=" + tableName2.getNameAsString(),
|
||||
"--bulkload", tableName1.getNameAsString() });
|
||||
} else {
|
||||
code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
|
||||
copy, new String[] { "--new.name=" + tableName2.getNameAsString(),
|
||||
tableName1.getNameAsString() });
|
||||
}
|
||||
assertEquals("copy job failed", 0, code);
|
||||
|
||||
// verify the data was copied into table 2
|
||||
for (int i = 0; i < 10; i++) {
|
||||
Get g = new Get(Bytes.toBytes("row" + i));
|
||||
Result r = t2.get(g);
|
||||
assertEquals(1, r.size());
|
||||
assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
|
||||
assertEquals("compare row values between two tables",
|
||||
t1.getDescriptor().getValue("row" + i),
|
||||
t2.getDescriptor().getValue("row" + i));
|
||||
}
|
||||
|
||||
assertEquals("compare count of mob rows after table copy", MobTestUtil.countMobRows(t1),
|
||||
MobTestUtil.countMobRows(t2));
|
||||
assertEquals("compare count of mob row values between two tables",
|
||||
t1.getDescriptor().getValues().size(),
|
||||
t2.getDescriptor().getValues().size());
|
||||
assertTrue("The mob row count is 0 but should be > 0",
|
||||
MobTestUtil.countMobRows(t2) > 0);
|
||||
|
||||
} finally {
|
||||
TEST_UTIL.deleteTable(tableName1);
|
||||
TEST_UTIL.deleteTable(tableName2);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple end-to-end test
|
||||
* @throws Exception
|
||||
*/
|
||||
@Test
|
||||
public void testCopyTable() throws Exception {
|
||||
|
|
@ -138,56 +208,70 @@ public class TestCopyTable {
|
|||
doCopyTableTest(true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple end-to-end test on table with MOB
|
||||
*/
|
||||
@Test
|
||||
public void testCopyTableWithMob() throws Exception {
|
||||
doCopyTableTestWithMob(false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple end-to-end test with bulkload on table with MOB.
|
||||
*/
|
||||
@Test
|
||||
public void testCopyTableWithBulkloadWithMob() throws Exception {
|
||||
doCopyTableTestWithMob(true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStartStopRow() throws Exception {
|
||||
final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
|
||||
final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
|
||||
final byte[] FAMILY = Bytes.toBytes("family");
|
||||
final byte[] COLUMN1 = Bytes.toBytes("c1");
|
||||
final byte[] ROW0 = Bytes.toBytesBinary("\\x01row0");
|
||||
final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1");
|
||||
final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2");
|
||||
final byte[] row0 = Bytes.toBytesBinary("\\x01row0");
|
||||
final byte[] row1 = Bytes.toBytesBinary("\\x01row1");
|
||||
final byte[] row2 = Bytes.toBytesBinary("\\x01row2");
|
||||
|
||||
Table t1 = TEST_UTIL.createTable(tableName1, FAMILY);
|
||||
Table t2 = TEST_UTIL.createTable(tableName2, FAMILY);
|
||||
try (Table t1 = TEST_UTIL.createTable(tableName1, FAMILY);
|
||||
Table t2 = TEST_UTIL.createTable(tableName2, FAMILY)) {
|
||||
|
||||
// put rows into the first table
|
||||
Put p = new Put(ROW0);
|
||||
p.addColumn(FAMILY, COLUMN1, COLUMN1);
|
||||
t1.put(p);
|
||||
p = new Put(ROW1);
|
||||
p.addColumn(FAMILY, COLUMN1, COLUMN1);
|
||||
t1.put(p);
|
||||
p = new Put(ROW2);
|
||||
p.addColumn(FAMILY, COLUMN1, COLUMN1);
|
||||
t1.put(p);
|
||||
// put rows into the first table
|
||||
Put p = new Put(row0);
|
||||
p.addColumn(FAMILY, COLUMN1, COLUMN1);
|
||||
t1.put(p);
|
||||
p = new Put(row1);
|
||||
p.addColumn(FAMILY, COLUMN1, COLUMN1);
|
||||
t1.put(p);
|
||||
p = new Put(row2);
|
||||
p.addColumn(FAMILY, COLUMN1, COLUMN1);
|
||||
t1.put(p);
|
||||
|
||||
CopyTable copy = new CopyTable();
|
||||
assertEquals(
|
||||
0,
|
||||
ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
|
||||
copy, new String[] { "--new.name=" + tableName2, "--startrow=\\x01row1",
|
||||
"--stoprow=\\x01row2", tableName1.getNameAsString() }));
|
||||
CopyTable copy = new CopyTable();
|
||||
assertEquals(0, ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()),
|
||||
copy, new String[]{"--new.name=" + tableName2, "--startrow=\\x01row1",
|
||||
"--stoprow=\\x01row2", tableName1.getNameAsString()}));
|
||||
|
||||
// verify the data was copied into table 2
|
||||
// row1 exist, row0, row2 do not exist
|
||||
Get g = new Get(ROW1);
|
||||
Result r = t2.get(g);
|
||||
assertEquals(1, r.size());
|
||||
assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
|
||||
// verify the data was copied into table 2
|
||||
// row1 exist, row0, row2 do not exist
|
||||
Get g = new Get(row1);
|
||||
Result r = t2.get(g);
|
||||
assertEquals(1, r.size());
|
||||
assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
|
||||
|
||||
g = new Get(ROW0);
|
||||
r = t2.get(g);
|
||||
assertEquals(0, r.size());
|
||||
g = new Get(row0);
|
||||
r = t2.get(g);
|
||||
assertEquals(0, r.size());
|
||||
|
||||
g = new Get(ROW2);
|
||||
r = t2.get(g);
|
||||
assertEquals(0, r.size());
|
||||
g = new Get(row2);
|
||||
r = t2.get(g);
|
||||
assertEquals(0, r.size());
|
||||
|
||||
t1.close();
|
||||
t2.close();
|
||||
TEST_UTIL.deleteTable(tableName1);
|
||||
TEST_UTIL.deleteTable(tableName2);
|
||||
} finally {
|
||||
TEST_UTIL.deleteTable(tableName1);
|
||||
TEST_UTIL.deleteTable(tableName2);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -215,8 +299,8 @@ public class TestCopyTable {
|
|||
|
||||
long currentTime = System.currentTimeMillis();
|
||||
String[] args = new String[] { "--new.name=" + targetTable, "--families=a:b", "--all.cells",
|
||||
"--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000),
|
||||
"--versions=1", sourceTable.getNameAsString() };
|
||||
"--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000),
|
||||
"--versions=1", sourceTable.getNameAsString() };
|
||||
assertNull(t2.get(new Get(ROW1)).getRow());
|
||||
|
||||
assertTrue(runCopy(args));
|
||||
|
|
|
|||
|
|
@ -2310,12 +2310,14 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility {
|
|||
|
||||
/**
|
||||
* Return the number of rows in the given table.
|
||||
* @param table to count rows
|
||||
* @return count of rows
|
||||
*/
|
||||
public int countRows(final Table table) throws IOException {
|
||||
public static int countRows(final Table table) throws IOException {
|
||||
return countRows(table, new Scan());
|
||||
}
|
||||
|
||||
public int countRows(final Table table, final Scan scan) throws IOException {
|
||||
public static int countRows(final Table table, final Scan scan) throws IOException {
|
||||
try (ResultScanner results = table.getScanner(scan)) {
|
||||
int count = 0;
|
||||
while (results.next() != null) {
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import java.util.Random;
|
|||
|
||||
import org.apache.hadoop.hbase.Cell;
|
||||
import org.apache.hadoop.hbase.CellUtil;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.KeyValue;
|
||||
import org.apache.hadoop.hbase.client.Result;
|
||||
import org.apache.hadoop.hbase.client.ResultScanner;
|
||||
|
|
@ -104,4 +105,16 @@ public class MobTestUtil {
|
|||
results.close();
|
||||
Assert.assertEquals(expectedCount, count);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the number of rows in the given table.
|
||||
* @param table to get the scanner
|
||||
* @return the number of rows
|
||||
*/
|
||||
public static int countMobRows(final Table table) throws IOException {
|
||||
Scan scan = new Scan();
|
||||
// Do not retrieve the mob data when scanning
|
||||
scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
|
||||
return HBaseTestingUtility.countRows(table, scan);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ import org.apache.hadoop.hbase.io.hfile.HFile;
|
|||
import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner;
|
||||
import org.apache.hadoop.hbase.mob.MobConstants;
|
||||
import org.apache.hadoop.hbase.mob.MobFileName;
|
||||
import org.apache.hadoop.hbase.mob.MobTestUtil;
|
||||
import org.apache.hadoop.hbase.mob.MobUtils;
|
||||
import org.apache.hadoop.hbase.regionserver.BloomType;
|
||||
import org.apache.hadoop.hbase.regionserver.HRegion;
|
||||
|
|
@ -295,7 +296,7 @@ public class TestMobCompactor {
|
|||
int rowNumPerRegion = count * rowNumPerFile;
|
||||
|
||||
assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion,
|
||||
countMobRows(table));
|
||||
MobTestUtil.countMobRows(table));
|
||||
assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion,
|
||||
countMobCells(table));
|
||||
assertEquals("Before deleting: mob file count", regionNum * count,
|
||||
|
|
@ -305,7 +306,7 @@ public class TestMobCompactor {
|
|||
createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
|
||||
|
||||
assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
|
||||
countMobRows(table));
|
||||
MobTestUtil.countMobRows(table));
|
||||
assertEquals("Before compaction: mob cells count", regionNum
|
||||
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
|
||||
assertEquals("Before compaction: family1 mob file count", regionNum * count,
|
||||
|
|
@ -322,7 +323,7 @@ public class TestMobCompactor {
|
|||
compactor.compact();
|
||||
|
||||
assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
|
||||
countMobRows(table));
|
||||
MobTestUtil.countMobRows(table));
|
||||
assertEquals("After compaction: mob cells count", regionNum
|
||||
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
|
||||
// After the compaction, the files smaller than the mob compaction merge size
|
||||
|
|
@ -445,7 +446,7 @@ public class TestMobCompactor {
|
|||
createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
|
||||
|
||||
assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
|
||||
countMobRows(table));
|
||||
MobTestUtil.countMobRows(table));
|
||||
assertEquals("Before compaction: mob cells count", regionNum
|
||||
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
|
||||
assertEquals("Before compaction: family1 mob file count", regionNum * count,
|
||||
|
|
@ -462,7 +463,7 @@ public class TestMobCompactor {
|
|||
compactor.compact();
|
||||
|
||||
assertEquals("After first compaction: mob rows count", regionNum
|
||||
* (rowNumPerRegion - delRowNum), countMobRows(table));
|
||||
* (rowNumPerRegion - delRowNum), MobTestUtil.countMobRows(table));
|
||||
assertEquals("After first compaction: mob cells count", regionNum
|
||||
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
|
||||
assertEquals("After first compaction: family1 mob file count", regionNum,
|
||||
|
|
@ -482,7 +483,7 @@ public class TestMobCompactor {
|
|||
admin.enableTable(tableName);
|
||||
|
||||
assertEquals("After restoring snapshot: mob rows count", regionNum * rowNumPerRegion,
|
||||
countMobRows(table));
|
||||
MobTestUtil.countMobRows(table));
|
||||
assertEquals("After restoring snapshot: mob cells count", regionNum * cellNumPerRow
|
||||
* rowNumPerRegion, countMobCells(table));
|
||||
assertEquals("After restoring snapshot: family1 mob file count", regionNum * count,
|
||||
|
|
@ -500,7 +501,7 @@ public class TestMobCompactor {
|
|||
compactor.compact();
|
||||
|
||||
assertEquals("After second compaction: mob rows count", regionNum * rowNumPerRegion,
|
||||
countMobRows(table));
|
||||
MobTestUtil.countMobRows(table));
|
||||
assertEquals("After second compaction: mob cells count", regionNum * cellNumPerRow
|
||||
* rowNumPerRegion, countMobCells(table));
|
||||
assertEquals("After second compaction: family1 mob file count", regionNum,
|
||||
|
|
@ -550,7 +551,7 @@ public class TestMobCompactor {
|
|||
int rowNumPerRegion = count * rowNumPerFile;
|
||||
|
||||
assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion,
|
||||
countMobRows(table));
|
||||
MobTestUtil.countMobRows(table));
|
||||
assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion,
|
||||
countMobCells(table));
|
||||
assertEquals("Before deleting: mob file count", regionNum * count,
|
||||
|
|
@ -559,7 +560,7 @@ public class TestMobCompactor {
|
|||
createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
|
||||
|
||||
assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
|
||||
countMobRows(table));
|
||||
MobTestUtil.countMobRows(table));
|
||||
assertEquals("Before compaction: mob cells count", regionNum
|
||||
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
|
||||
assertEquals("Before compaction: family1 mob file count", regionNum * count,
|
||||
|
|
@ -576,7 +577,7 @@ public class TestMobCompactor {
|
|||
|
||||
waitUntilMobCompactionFinished(tableName);
|
||||
assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
|
||||
countMobRows(table));
|
||||
MobTestUtil.countMobRows(table));
|
||||
assertEquals("After compaction: mob cells count", regionNum
|
||||
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
|
||||
assertEquals("After compaction: family1 mob file count", regionNum,
|
||||
|
|
@ -763,18 +764,6 @@ public class TestMobCompactor {
|
|||
assertEquals(CompactionState.NONE, state);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the number of rows in the given table.
|
||||
* @param table to get the scanner
|
||||
* @return the number of rows
|
||||
*/
|
||||
private int countMobRows(final Table table) throws IOException {
|
||||
Scan scan = new Scan();
|
||||
// Do not retrieve the mob data when scanning
|
||||
scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
|
||||
return TEST_UTIL.countRows(table, scan);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the number of cells in the given table.
|
||||
* @param table to get the scanner
|
||||
|
|
@ -923,13 +912,15 @@ public class TestMobCompactor {
|
|||
|
||||
for (int i = 0; i < 1000; i ++) {
|
||||
Put put0 = new Put(Bytes.toBytes("r0" + i));
|
||||
put0.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), tsFor20151130Monday, Bytes.toBytes(mobValue0));
|
||||
put0.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1),
|
||||
tsFor20151130Monday, Bytes.toBytes(mobValue0));
|
||||
pArray[i] = put0;
|
||||
}
|
||||
loadData(admin, bufMut, tableName, pArray);
|
||||
|
||||
Put put06 = new Put(mobKey06);
|
||||
put06.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), tsFor20151128Saturday, Bytes.toBytes(mobValue0));
|
||||
put06.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1),
|
||||
tsFor20151128Saturday, Bytes.toBytes(mobValue0));
|
||||
|
||||
loadData(admin, bufMut, tableName, new Put[] { put06 });
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue