HBASE-19895 Add keepDeletedCells option in ScanOptions for customizing scanInfo in pre-hooks (Ankit Singhal)

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CustomizedScanInfoBuilder.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.regionserver;
 
+import org.apache.hadoop.hbase.KeepDeletedCells;
 import org.apache.yetus.audience.InterfaceAudience;
 
 /**
@@ -31,6 +32,8 @@ public class CustomizedScanInfoBuilder implements ScanOptions {
 
   private Long ttl;
 
+  private KeepDeletedCells keepDeletedCells = null;
+
   public CustomizedScanInfoBuilder(ScanInfo scanInfo) {
     this.scanInfo = scanInfo;
   }
@@ -56,14 +59,25 @@ public class CustomizedScanInfoBuilder implements ScanOptions {
   }
 
   public ScanInfo build() {
-    if (maxVersions == null && ttl == null) {
+    if (maxVersions == null && ttl == null && keepDeletedCells == null) {
       return scanInfo;
     }
-    return scanInfo.customize(getMaxVersions(), getTTL());
+    return scanInfo.customize(getMaxVersions(), getTTL(), getKeepDeletedCells());
   }
 
   @Override
   public String toString() {
     return "ScanOptions [maxVersions=" + getMaxVersions() + ", TTL=" + getTTL() + "]";
   }
+
+  @Override
+  public void setKeepDeletedCells(KeepDeletedCells keepDeletedCells) {
+    this.keepDeletedCells = keepDeletedCells;
+  }
+
+  @Override
+  public KeepDeletedCells getKeepDeletedCells() {
+    return keepDeletedCells != null ? keepDeletedCells : scanInfo.getKeepDeletedCells();
+  }
+
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanInfo.java

@@ -171,10 +171,11 @@ public class ScanInfo {
   }
 
   /**
-   * Used for CP users for customizing max versions and ttl.
+   * Used for CP users for customizing max versions, ttl and keepDeletedCells.
    */
-  ScanInfo customize(int maxVersions, long ttl) {
+  ScanInfo customize(int maxVersions, long ttl, KeepDeletedCells keepDeletedCells) {
-    return new ScanInfo(family, minVersions, maxVersions, ttl, keepDeletedCells, ttl, comparator,
+    return new ScanInfo(family, minVersions, maxVersions, ttl, keepDeletedCells, timeToPurgeDeletes,
-        ttl, usePread, maxVersions, parallelSeekEnabled, ttl, newVersionBehavior);
+        comparator, tableMaxRowSize, usePread, cellsPerTimeoutCheck, parallelSeekEnabled,
+        preadMaxBytes, newVersionBehavior);
   }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanOptions.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
+import org.apache.hadoop.hbase.KeepDeletedCells;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.apache.yetus.audience.InterfaceStability;
 
@@ -59,4 +60,8 @@ public interface ScanOptions {
   long getTTL();
 
   void setTTL(long ttl);
+
+  void setKeepDeletedCells(KeepDeletedCells keepDeletedCells);
+
+  KeepDeletedCells getKeepDeletedCells();
 }

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java

@@ -218,7 +218,7 @@ public class TestCompaction {
       final int ttl = 1000;
       for (HStore store : this.r.stores.values()) {
         ScanInfo old = store.getScanInfo();
-        ScanInfo si = old.customize(old.getMaxVersions(), ttl);
+        ScanInfo si = old.customize(old.getMaxVersions(), ttl, old.getKeepDeletedCells());
         store.setScanInfo(si);
       }
       Thread.sleep(ttl);

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java

@@ -170,7 +170,8 @@ public class TestDefaultCompactSelection extends TestCompactionPolicy {
   public void testCompactionEmptyHFile() throws IOException {
     // Set TTL
     ScanInfo oldScanInfo = store.getScanInfo();
-    ScanInfo newScanInfo = oldScanInfo.customize(oldScanInfo.getMaxVersions(), 600);
+    ScanInfo newScanInfo = oldScanInfo.customize(oldScanInfo.getMaxVersions(), 600,
+        oldScanInfo.getKeepDeletedCells());
     store.setScanInfo(newScanInfo);
     // Do not compact empty store file
     List<HStoreFile> candidates = sfCreate(0);

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMajorCompaction.java

@@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.HBaseTestCase;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeepDeletedCells;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.Result;
@@ -134,36 +135,20 @@ public class TestMajorCompaction {
    * Test that on a major compaction, if all cells are expired or deleted, then
    * we'll end up with no product.  Make sure scanner over region returns
    * right answer in this case - and that it just basically works.
-   * @throws IOException
+   * @throws IOException exception encountered
    */
   @Test
   public void testMajorCompactingToNoOutput() throws IOException {
-    createStoreFile(r);
+    testMajorCompactingWithDeletes(KeepDeletedCells.FALSE);
-    for (int i = 0; i < compactionThreshold; i++) {
+  }
-      createStoreFile(r);
+
-    }
+  /**
-    // Now delete everything.
+   * Test that on a major compaction,Deleted cells are retained if keep deleted cells is set to true
-    InternalScanner s = r.getScanner(new Scan());
+   * @throws IOException exception encountered
-    do {
+   */
-      List<Cell> results = new ArrayList<>();
+  @Test
-      boolean result = s.next(results);
+  public void testMajorCompactingWithKeepDeletedCells() throws IOException {
-      r.delete(new Delete(CellUtil.cloneRow(results.get(0))));
+    testMajorCompactingWithDeletes(KeepDeletedCells.TRUE);
-      if (!result) break;
-    } while(true);
-    s.close();
-    // Flush
-    r.flush(true);
-    // Major compact.
-    r.compact(true);
-    s = r.getScanner(new Scan());
-    int counter = 0;
-    do {
-      List<Cell> results = new ArrayList<>();
-      boolean result = s.next(results);
-      if (!result) break;
-      counter++;
-    } while(true);
-    assertEquals(0, counter);
   }
 
   /**
@@ -298,7 +283,7 @@ public class TestMajorCompaction {
     final int ttl = 1000;
     for (HStore store : r.getStores()) {
       ScanInfo old = store.getScanInfo();
-      ScanInfo si = old.customize(old.getMaxVersions(), ttl);
+      ScanInfo si = old.customize(old.getMaxVersions(), ttl, old.getKeepDeletedCells());
       store.setScanInfo(si);
     }
     Thread.sleep(1000);
@@ -471,7 +456,9 @@ public class TestMajorCompaction {
       boolean result = s.next(results);
       assertTrue(!results.isEmpty());
       r.delete(new Delete(CellUtil.cloneRow(results.get(0))));
-      if (!result) break;
+      if (!result) {
+        break;
+      }
     } while (true);
     s.close();
     // Flush
@@ -485,10 +472,51 @@ public class TestMajorCompaction {
     do {
       List<Cell> results = new ArrayList<>();
       boolean result = s.next(results);
-      if (!result) break;
+      if (!result) {
+        break;
+      }
       counter++;
     } while (true);
     s.close();
     assertEquals(0, counter);
   }
+
+  private void testMajorCompactingWithDeletes(KeepDeletedCells keepDeletedCells)
+      throws IOException {
+    createStoreFile(r);
+    for (int i = 0; i < compactionThreshold; i++) {
+      createStoreFile(r);
+    }
+    // Now delete everything.
+    InternalScanner s = r.getScanner(new Scan());
+    int originalCount = 0;
+    do {
+      List<Cell> results = new ArrayList<>();
+      boolean result = s.next(results);
+      r.delete(new Delete(CellUtil.cloneRow(results.get(0))));
+      if (!result) break;
+      originalCount++;
+    } while (true);
+    s.close();
+    // Flush
+    r.flush(true);
+
+    for (HStore store : this.r.stores.values()) {
+      ScanInfo old = store.getScanInfo();
+      ScanInfo si = old.customize(old.getMaxVersions(), old.getTtl(), keepDeletedCells);
+      store.setScanInfo(si);
+    }
+    // Major compact.
+    r.compact(true);
+    s = r.getScanner(new Scan().setRaw(true));
+    int counter = 0;
+    do {
+      List<Cell> results = new ArrayList<>();
+      boolean result = s.next(results);
+      if (!result) break;
+      counter++;
+    } while (true);
+    assertEquals(keepDeletedCells == KeepDeletedCells.TRUE ? originalCount : 0, counter);
+
+  }
 }