diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java index 6f416710e1e..ed6857dd3b6 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java @@ -30,15 +30,20 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.mob.MobTestUtil; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MapReduceTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.LauncherSecurityManager; import org.apache.hadoop.util.ToolRunner; + import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; @@ -100,7 +105,7 @@ public class TestCopyTable { if (bulkload) { code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy, new String[] { "--new.name=" + tableName2.getNameAsString(), - "--bulkload", tableName1.getNameAsString() }); + "--bulkload", tableName1.getNameAsString() }); } else { code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), copy, new String[] { "--new.name=" + tableName2.getNameAsString(), @@ -121,9 +126,74 @@ public class TestCopyTable { } } + private void doCopyTableTestWithMob(boolean bulkload) throws Exception { + final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); + final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); + final byte[] FAMILY = Bytes.toBytes("mob"); + final byte[] COLUMN1 = Bytes.toBytes("c1"); + + ColumnFamilyDescriptorBuilder cfd = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY); + + cfd.setMobEnabled(true); + cfd.setMobThreshold(5); + TableDescriptor desc1 = TableDescriptorBuilder.newBuilder(tableName1) + .setColumnFamily(cfd.build()) + .build(); + TableDescriptor desc2 = TableDescriptorBuilder.newBuilder(tableName2) + .setColumnFamily(cfd.build()) + .build(); + + try (Table t1 = TEST_UTIL.createTable(desc1, null); + Table t2 = TEST_UTIL.createTable(desc2, null);) { + + // put rows into the first table + for (int i = 0; i < 10; i++) { + Put p = new Put(Bytes.toBytes("row" + i)); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + t1.put(p); + } + + CopyTable copy = new CopyTable(); + + int code; + if (bulkload) { + code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), + copy, new String[] { "--new.name=" + tableName2.getNameAsString(), + "--bulkload", tableName1.getNameAsString() }); + } else { + code = ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), + copy, new String[] { "--new.name=" + tableName2.getNameAsString(), + tableName1.getNameAsString() }); + } + assertEquals("copy job failed", 0, code); + + // verify the data was copied into table 2 + for (int i = 0; i < 10; i++) { + Get g = new Get(Bytes.toBytes("row" + i)); + Result r = t2.get(g); + assertEquals(1, r.size()); + assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); + assertEquals("compare row values between two tables", + t1.getDescriptor().getValue("row" + i), + t2.getDescriptor().getValue("row" + i)); + } + + assertEquals("compare count of mob rows after table copy", MobTestUtil.countMobRows(t1), + MobTestUtil.countMobRows(t2)); + assertEquals("compare count of mob row values between two tables", + t1.getDescriptor().getValues().size(), + t2.getDescriptor().getValues().size()); + assertTrue("The mob row count is 0 but should be > 0", + MobTestUtil.countMobRows(t2) > 0); + + } finally { + TEST_UTIL.deleteTable(tableName1); + TEST_UTIL.deleteTable(tableName2); + } + } + /** * Simple end-to-end test - * @throws Exception */ @Test public void testCopyTable() throws Exception { @@ -138,56 +208,70 @@ public class TestCopyTable { doCopyTableTest(true); } + /** + * Simple end-to-end test on table with MOB + */ + @Test + public void testCopyTableWithMob() throws Exception { + doCopyTableTestWithMob(false); + } + + /** + * Simple end-to-end test with bulkload on table with MOB. + */ + @Test + public void testCopyTableWithBulkloadWithMob() throws Exception { + doCopyTableTestWithMob(true); + } + @Test public void testStartStopRow() throws Exception { final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1"); final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2"); final byte[] FAMILY = Bytes.toBytes("family"); final byte[] COLUMN1 = Bytes.toBytes("c1"); - final byte[] ROW0 = Bytes.toBytesBinary("\\x01row0"); - final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1"); - final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2"); + final byte[] row0 = Bytes.toBytesBinary("\\x01row0"); + final byte[] row1 = Bytes.toBytesBinary("\\x01row1"); + final byte[] row2 = Bytes.toBytesBinary("\\x01row2"); - Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); - Table t2 = TEST_UTIL.createTable(tableName2, FAMILY); + try (Table t1 = TEST_UTIL.createTable(tableName1, FAMILY); + Table t2 = TEST_UTIL.createTable(tableName2, FAMILY)) { - // put rows into the first table - Put p = new Put(ROW0); - p.addColumn(FAMILY, COLUMN1, COLUMN1); - t1.put(p); - p = new Put(ROW1); - p.addColumn(FAMILY, COLUMN1, COLUMN1); - t1.put(p); - p = new Put(ROW2); - p.addColumn(FAMILY, COLUMN1, COLUMN1); - t1.put(p); + // put rows into the first table + Put p = new Put(row0); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + t1.put(p); + p = new Put(row1); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + t1.put(p); + p = new Put(row2); + p.addColumn(FAMILY, COLUMN1, COLUMN1); + t1.put(p); - CopyTable copy = new CopyTable(); - assertEquals( - 0, - ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), - copy, new String[] { "--new.name=" + tableName2, "--startrow=\\x01row1", - "--stoprow=\\x01row2", tableName1.getNameAsString() })); + CopyTable copy = new CopyTable(); + assertEquals(0, ToolRunner.run(new Configuration(TEST_UTIL.getConfiguration()), + copy, new String[]{"--new.name=" + tableName2, "--startrow=\\x01row1", + "--stoprow=\\x01row2", tableName1.getNameAsString()})); - // verify the data was copied into table 2 - // row1 exist, row0, row2 do not exist - Get g = new Get(ROW1); - Result r = t2.get(g); - assertEquals(1, r.size()); - assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); + // verify the data was copied into table 2 + // row1 exist, row0, row2 do not exist + Get g = new Get(row1); + Result r = t2.get(g); + assertEquals(1, r.size()); + assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); - g = new Get(ROW0); - r = t2.get(g); - assertEquals(0, r.size()); + g = new Get(row0); + r = t2.get(g); + assertEquals(0, r.size()); - g = new Get(ROW2); - r = t2.get(g); - assertEquals(0, r.size()); + g = new Get(row2); + r = t2.get(g); + assertEquals(0, r.size()); - t1.close(); - t2.close(); - TEST_UTIL.deleteTable(tableName1); - TEST_UTIL.deleteTable(tableName2); + } finally { + TEST_UTIL.deleteTable(tableName1); + TEST_UTIL.deleteTable(tableName2); + } } /** @@ -215,8 +299,8 @@ public class TestCopyTable { long currentTime = System.currentTimeMillis(); String[] args = new String[] { "--new.name=" + targetTable, "--families=a:b", "--all.cells", - "--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000), - "--versions=1", sourceTable.getNameAsString() }; + "--starttime=" + (currentTime - 100000), "--endtime=" + (currentTime + 100000), + "--versions=1", sourceTable.getNameAsString() }; assertNull(t2.get(new Get(ROW1)).getRow()); assertTrue(runCopy(args)); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index ab9a7992ed5..695daeabbe4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -2310,12 +2310,14 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { /** * Return the number of rows in the given table. + * @param table to count rows + * @return count of rows */ - public int countRows(final Table table) throws IOException { + public static int countRows(final Table table) throws IOException { return countRows(table, new Scan()); } - public int countRows(final Table table, final Scan scan) throws IOException { + public static int countRows(final Table table, final Scan scan) throws IOException { try (ResultScanner results = table.getScanner(scan)) { int count = 0; while (results.next() != null) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java index 76cb6645966..8972dba58e8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/MobTestUtil.java @@ -24,6 +24,7 @@ import java.util.Random; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; @@ -104,4 +105,16 @@ public class MobTestUtil { results.close(); Assert.assertEquals(expectedCount, count); } + + /** + * Gets the number of rows in the given table. + * @param table to get the scanner + * @return the number of rows + */ + public static int countMobRows(final Table table) throws IOException { + Scan scan = new Scan(); + // Do not retrieve the mob data when scanning + scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); + return HBaseTestingUtility.countRows(table, scan); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java index 8ed4fbb1556..86e07d2d406 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestMobCompactor.java @@ -78,6 +78,7 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner; import org.apache.hadoop.hbase.mob.MobConstants; import org.apache.hadoop.hbase.mob.MobFileName; +import org.apache.hadoop.hbase.mob.MobTestUtil; import org.apache.hadoop.hbase.mob.MobUtils; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.regionserver.HRegion; @@ -295,7 +296,7 @@ public class TestMobCompactor { int rowNumPerRegion = count * rowNumPerFile; assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion, - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, countMobCells(table)); assertEquals("Before deleting: mob file count", regionNum * count, @@ -305,7 +306,7 @@ public class TestMobCompactor { createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("Before compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("Before compaction: family1 mob file count", regionNum * count, @@ -322,7 +323,7 @@ public class TestMobCompactor { compactor.compact(); assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("After compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); // After the compaction, the files smaller than the mob compaction merge size @@ -445,7 +446,7 @@ public class TestMobCompactor { createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("Before compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("Before compaction: family1 mob file count", regionNum * count, @@ -462,7 +463,7 @@ public class TestMobCompactor { compactor.compact(); assertEquals("After first compaction: mob rows count", regionNum - * (rowNumPerRegion - delRowNum), countMobRows(table)); + * (rowNumPerRegion - delRowNum), MobTestUtil.countMobRows(table)); assertEquals("After first compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("After first compaction: family1 mob file count", regionNum, @@ -482,7 +483,7 @@ public class TestMobCompactor { admin.enableTable(tableName); assertEquals("After restoring snapshot: mob rows count", regionNum * rowNumPerRegion, - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("After restoring snapshot: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, countMobCells(table)); assertEquals("After restoring snapshot: family1 mob file count", regionNum * count, @@ -500,7 +501,7 @@ public class TestMobCompactor { compactor.compact(); assertEquals("After second compaction: mob rows count", regionNum * rowNumPerRegion, - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("After second compaction: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, countMobCells(table)); assertEquals("After second compaction: family1 mob file count", regionNum, @@ -550,7 +551,7 @@ public class TestMobCompactor { int rowNumPerRegion = count * rowNumPerFile; assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion, - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion, countMobCells(table)); assertEquals("Before deleting: mob file count", regionNum * count, @@ -559,7 +560,7 @@ public class TestMobCompactor { createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1)); assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("Before compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("Before compaction: family1 mob file count", regionNum * count, @@ -576,7 +577,7 @@ public class TestMobCompactor { waitUntilMobCompactionFinished(tableName); assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum), - countMobRows(table)); + MobTestUtil.countMobRows(table)); assertEquals("After compaction: mob cells count", regionNum * (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table)); assertEquals("After compaction: family1 mob file count", regionNum, @@ -763,18 +764,6 @@ public class TestMobCompactor { assertEquals(CompactionState.NONE, state); } - /** - * Gets the number of rows in the given table. - * @param table to get the scanner - * @return the number of rows - */ - private int countMobRows(final Table table) throws IOException { - Scan scan = new Scan(); - // Do not retrieve the mob data when scanning - scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); - return TEST_UTIL.countRows(table, scan); - } - /** * Gets the number of cells in the given table. * @param table to get the scanner @@ -923,13 +912,15 @@ public class TestMobCompactor { for (int i = 0; i < 1000; i ++) { Put put0 = new Put(Bytes.toBytes("r0" + i)); - put0.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), tsFor20151130Monday, Bytes.toBytes(mobValue0)); + put0.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), + tsFor20151130Monday, Bytes.toBytes(mobValue0)); pArray[i] = put0; } loadData(admin, bufMut, tableName, pArray); Put put06 = new Put(mobKey06); - put06.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), tsFor20151128Saturday, Bytes.toBytes(mobValue0)); + put06.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1), + tsFor20151128Saturday, Bytes.toBytes(mobValue0)); loadData(admin, bufMut, tableName, new Put[] { put06 });