HBASE-23711 - Add test for MinVersions and KeepDeletedCells TTL (#1079)

Signed-off-by: stack <stack@apache.org>
Signed-off-by: Viraj Jasani <vjasani@apache.org>

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java

@@ -27,17 +27,23 @@ import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellUtil;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeepDeletedCells;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.filter.TimestampsFilter;
 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
 import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
+import org.junit.Assert;
 import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.Test;
@@ -54,7 +60,7 @@ public class TestMinVersions {
   public static final HBaseClassTestRule CLASS_RULE =
       HBaseClassTestRule.forClass(TestMinVersions.class);
 
-  HBaseTestingUtility hbu = HBaseTestingUtility.createLocalHTU();
+  HBaseTestingUtility hbu = new HBaseTestingUtility();
   private final byte[] T0 = Bytes.toBytes("0");
   private final byte[] T1 = Bytes.toBytes("1");
   private final byte[] T2 = Bytes.toBytes("2");
@@ -71,8 +77,14 @@ public class TestMinVersions {
    */
   @Test
   public void testGetClosestBefore() throws Exception {
-    HTableDescriptor htd =
+
-        hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, KeepDeletedCells.FALSE);
+    ColumnFamilyDescriptor cfd =
+      ColumnFamilyDescriptorBuilder.newBuilder(c0)
+      .setMinVersions(1).setMaxVersions(1000).setTimeToLive(1).
+        setKeepDeletedCells(KeepDeletedCells.FALSE).build();
+
+    TableDescriptor htd = TableDescriptorBuilder.
+      newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
     HRegion region = hbu.createLocalHRegion(htd, null, null);
     try {
 
@@ -121,8 +133,15 @@ public class TestMinVersions {
   @Test
   public void testStoreMemStore() throws Exception {
     // keep 3 versions minimum
-    HTableDescriptor htd =
+
-        hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE);
+    ColumnFamilyDescriptor cfd =
+      ColumnFamilyDescriptorBuilder.newBuilder(c0)
+        .setMinVersions(3).setMaxVersions(1000).setTimeToLive(1).
+        setKeepDeletedCells(KeepDeletedCells.FALSE).build();
+
+    TableDescriptor htd = TableDescriptorBuilder.
+      newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
+
     HRegion region = hbu.createLocalHRegion(htd, null, null);
     // 2s in the past
     long ts = EnvironmentEdgeManager.currentTime() - 2000;
@@ -176,8 +195,14 @@ public class TestMinVersions {
    */
   @Test
   public void testDelete() throws Exception {
-    HTableDescriptor htd =
+    ColumnFamilyDescriptor cfd =
-        hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE);
+      ColumnFamilyDescriptorBuilder.newBuilder(c0)
+        .setMinVersions(3).setMaxVersions(1000).setTimeToLive(1).
+        setKeepDeletedCells(KeepDeletedCells.FALSE).build();
+
+    TableDescriptor htd = TableDescriptorBuilder.
+      newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
+
     HRegion region = hbu.createLocalHRegion(htd, null, null);
 
     // 2s in the past
@@ -235,8 +260,13 @@ public class TestMinVersions {
    */
   @Test
   public void testMemStore() throws Exception {
-    HTableDescriptor htd =
+    ColumnFamilyDescriptor cfd =
-        hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
+      ColumnFamilyDescriptorBuilder.newBuilder(c0)
+        .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
+        setKeepDeletedCells(KeepDeletedCells.FALSE).build();
+
+    TableDescriptor htd = TableDescriptorBuilder.
+      newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
     HRegion region = hbu.createLocalHRegion(htd, null, null);
 
     // 2s in the past
@@ -310,9 +340,14 @@ public class TestMinVersions {
    */
   @Test
   public void testBaseCase() throws Exception {
-    // 1 version minimum, 1000 versions maximum, ttl = 1s
+    // 2 version minimum, 1000 versions maximum, ttl = 1s
-    HTableDescriptor htd =
+    ColumnFamilyDescriptor cfd =
-        hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
+      ColumnFamilyDescriptorBuilder.newBuilder(c0)
+        .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
+        setKeepDeletedCells(KeepDeletedCells.FALSE).build();
+
+    TableDescriptor htd = TableDescriptorBuilder.
+      newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
     HRegion region = hbu.createLocalHRegion(htd, null, null);
     try {
 
@@ -403,10 +438,23 @@ public class TestMinVersions {
    */
   @Test
   public void testFilters() throws Exception {
-    HTableDescriptor htd =
-        hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
-    HRegion region = hbu.createLocalHRegion(htd, null, null);
     final byte [] c1 = COLUMNS[1];
+    ColumnFamilyDescriptor cfd =
+      ColumnFamilyDescriptorBuilder.newBuilder(c0)
+        .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
+        setKeepDeletedCells(KeepDeletedCells.FALSE).build();
+
+    ColumnFamilyDescriptor cfd2 =
+      ColumnFamilyDescriptorBuilder.newBuilder(c1)
+        .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
+        setKeepDeletedCells(KeepDeletedCells.FALSE).build();
+    List<ColumnFamilyDescriptor> cfdList = new ArrayList();
+    cfdList.add(cfd);
+    cfdList.add(cfd2);
+
+    TableDescriptor htd = TableDescriptorBuilder.
+      newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamilies(cfdList).build();
+    HRegion region = hbu.createLocalHRegion(htd, null, null);
 
     // 2s in the past
     long ts = EnvironmentEdgeManager.currentTime() - 2000;
@@ -476,12 +524,96 @@ public class TestMinVersions {
     }
   }
 
+  @Test
+  public void testMinVersionsWithKeepDeletedCellsTTL() throws Exception {
+    int ttl = 4;
+    ColumnFamilyDescriptor cfd =
+      ColumnFamilyDescriptorBuilder.newBuilder(c0)
+        .setMinVersions(2).setMaxVersions(Integer.MAX_VALUE).setTimeToLive(ttl).
+        setKeepDeletedCells(KeepDeletedCells.TTL).build();
+
+    TableDescriptor htd = TableDescriptorBuilder.
+      newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
+
+    HRegion region = hbu.createLocalHRegion(htd, null, null);
+
+    long startTS = EnvironmentEdgeManager.currentTime();
+    ManualEnvironmentEdge injectEdge = new ManualEnvironmentEdge();
+    injectEdge.setValue(startTS);
+    EnvironmentEdgeManager.injectEdge(injectEdge);
+
+    long ts = startTS - 2000;
+    // 1st version
+    Put p = new Put(T1, ts-3);
+    p.addColumn(c0, c0, T1);
+    region.put(p);
+
+    // 2nd version
+    p = new Put(T1, ts-2);
+    p.addColumn(c0, c0, T2);
+    region.put(p);
+
+    // 3rd version
+    p = new Put(T1, ts-1);
+    p.addColumn(c0, c0, T3);
+    region.put(p);
+
+    Get g;
+    Result r;
+
+    //check we can still see all versions before compaction
+    g = new Get(T1);
+    g.readAllVersions();
+    g.setTimeRange(0, ts);
+    r = region.get(g);
+    checkResult(r, c0, T3, T2, T1);
+
+    region.flush(true);
+    region.compact(true);
+    Assert.assertEquals(startTS, EnvironmentEdgeManager.currentTime());
+    long expiredTime = EnvironmentEdgeManager.currentTime() - ts - 3;
+    Assert.assertTrue("TTL for T1 has expired", expiredTime < (ttl * 1000));
+    //check that nothing was purged yet
+    g = new Get(T1);
+    g.readAllVersions();
+    g.setTimeRange(0, ts);
+    r = region.get(g);
+    checkResult(r, c0, T3, T2, T1);
+
+    g = new Get(T1);
+    g.readAllVersions();
+    g.setTimeRange(0, ts -1);
+    r = region.get(g);
+    checkResult(r, c0, T2, T1);
+
+    injectEdge.incValue(ttl * 1000);
+
+    region.flush(true);
+    region.compact(true);
+
+    //check that after compaction (which is after TTL) that only T1 was purged
+    g = new Get(T1);
+    g.readAllVersions();
+    g.setTimeRange(0, ts);
+    r = region.get(g);
+    checkResult(r, c0, T3, T2);
+
+    g = new Get(T1);
+    g.readAllVersions();
+    g.setTimestamp(ts -2);
+    r = region.get(g);
+    checkResult(r, c0, T2);
+  }
+
   private void checkResult(Result r, byte[] col, byte[] ... vals) {
-    assertEquals(r.size(), vals.length);
+    assertEquals(vals.length, r.size());
     List<Cell> kvs = r.getColumnCells(col, col);
     assertEquals(kvs.size(), vals.length);
     for (int i=0;i<vals.length;i++) {
-      assertTrue(CellUtil.matchingValue(kvs.get(i), vals[i]));
+      String expected = Bytes.toString(vals[i]);
+      String actual = Bytes.toString(CellUtil.cloneValue(kvs.get(i)));
+      assertTrue(expected + " was expected but doesn't match " + actual,
+          CellUtil.matchingValue(kvs.get(i), vals[i]));
     }
   }