HBASE-19895 Add keepDeletedCells option in ScanOptions for customizing scanInfo in pre-hooks (Ankit Singhal)

This commit is contained in:
tedyu 2018-02-01 18:28:05 -08:00
parent a165bd766a
commit efed668f74
6 changed files with 87 additions and 38 deletions

View File

@ -17,6 +17,7 @@
*/ */
package org.apache.hadoop.hbase.regionserver; package org.apache.hadoop.hbase.regionserver;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceAudience;
/** /**
@ -31,6 +32,8 @@ public class CustomizedScanInfoBuilder implements ScanOptions {
private Long ttl; private Long ttl;
private KeepDeletedCells keepDeletedCells = null;
public CustomizedScanInfoBuilder(ScanInfo scanInfo) { public CustomizedScanInfoBuilder(ScanInfo scanInfo) {
this.scanInfo = scanInfo; this.scanInfo = scanInfo;
} }
@ -56,14 +59,25 @@ public class CustomizedScanInfoBuilder implements ScanOptions {
} }
public ScanInfo build() { public ScanInfo build() {
if (maxVersions == null && ttl == null) { if (maxVersions == null && ttl == null && keepDeletedCells == null) {
return scanInfo; return scanInfo;
} }
return scanInfo.customize(getMaxVersions(), getTTL()); return scanInfo.customize(getMaxVersions(), getTTL(), getKeepDeletedCells());
} }
@Override @Override
public String toString() { public String toString() {
return "ScanOptions [maxVersions=" + getMaxVersions() + ", TTL=" + getTTL() + "]"; return "ScanOptions [maxVersions=" + getMaxVersions() + ", TTL=" + getTTL() + "]";
} }
@Override
public void setKeepDeletedCells(KeepDeletedCells keepDeletedCells) {
this.keepDeletedCells = keepDeletedCells;
}
@Override
public KeepDeletedCells getKeepDeletedCells() {
return keepDeletedCells != null ? keepDeletedCells : scanInfo.getKeepDeletedCells();
}
} }

View File

@ -171,10 +171,11 @@ public class ScanInfo {
} }
/** /**
* Used for CP users for customizing max versions and ttl. * Used for CP users for customizing max versions, ttl and keepDeletedCells.
*/ */
ScanInfo customize(int maxVersions, long ttl) { ScanInfo customize(int maxVersions, long ttl, KeepDeletedCells keepDeletedCells) {
return new ScanInfo(family, minVersions, maxVersions, ttl, keepDeletedCells, ttl, comparator, return new ScanInfo(family, minVersions, maxVersions, ttl, keepDeletedCells, timeToPurgeDeletes,
ttl, usePread, maxVersions, parallelSeekEnabled, ttl, newVersionBehavior); comparator, tableMaxRowSize, usePread, cellsPerTimeoutCheck, parallelSeekEnabled,
preadMaxBytes, newVersionBehavior);
} }
} }

View File

@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.regionserver; package org.apache.hadoop.hbase.regionserver;
import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability; import org.apache.yetus.audience.InterfaceStability;
@ -59,4 +60,8 @@ public interface ScanOptions {
long getTTL(); long getTTL();
void setTTL(long ttl); void setTTL(long ttl);
void setKeepDeletedCells(KeepDeletedCells keepDeletedCells);
KeepDeletedCells getKeepDeletedCells();
} }

View File

@ -218,7 +218,7 @@ public class TestCompaction {
final int ttl = 1000; final int ttl = 1000;
for (HStore store : this.r.stores.values()) { for (HStore store : this.r.stores.values()) {
ScanInfo old = store.getScanInfo(); ScanInfo old = store.getScanInfo();
ScanInfo si = old.customize(old.getMaxVersions(), ttl); ScanInfo si = old.customize(old.getMaxVersions(), ttl, old.getKeepDeletedCells());
store.setScanInfo(si); store.setScanInfo(si);
} }
Thread.sleep(ttl); Thread.sleep(ttl);

View File

@ -170,7 +170,8 @@ public class TestDefaultCompactSelection extends TestCompactionPolicy {
public void testCompactionEmptyHFile() throws IOException { public void testCompactionEmptyHFile() throws IOException {
// Set TTL // Set TTL
ScanInfo oldScanInfo = store.getScanInfo(); ScanInfo oldScanInfo = store.getScanInfo();
ScanInfo newScanInfo = oldScanInfo.customize(oldScanInfo.getMaxVersions(), 600); ScanInfo newScanInfo = oldScanInfo.customize(oldScanInfo.getMaxVersions(), 600,
oldScanInfo.getKeepDeletedCells());
store.setScanInfo(newScanInfo); store.setScanInfo(newScanInfo);
// Do not compact empty store file // Do not compact empty store file
List<HStoreFile> candidates = sfCreate(0); List<HStoreFile> candidates = sfCreate(0);

View File

@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Result;
@ -134,36 +135,20 @@ public class TestMajorCompaction {
* Test that on a major compaction, if all cells are expired or deleted, then * Test that on a major compaction, if all cells are expired or deleted, then
* we'll end up with no product. Make sure scanner over region returns * we'll end up with no product. Make sure scanner over region returns
* right answer in this case - and that it just basically works. * right answer in this case - and that it just basically works.
* @throws IOException * @throws IOException exception encountered
*/ */
@Test @Test
public void testMajorCompactingToNoOutput() throws IOException { public void testMajorCompactingToNoOutput() throws IOException {
createStoreFile(r); testMajorCompactingWithDeletes(KeepDeletedCells.FALSE);
for (int i = 0; i < compactionThreshold; i++) {
createStoreFile(r);
} }
// Now delete everything.
InternalScanner s = r.getScanner(new Scan()); /**
do { * Test that on a major compaction,Deleted cells are retained if keep deleted cells is set to true
List<Cell> results = new ArrayList<>(); * @throws IOException exception encountered
boolean result = s.next(results); */
r.delete(new Delete(CellUtil.cloneRow(results.get(0)))); @Test
if (!result) break; public void testMajorCompactingWithKeepDeletedCells() throws IOException {
} while(true); testMajorCompactingWithDeletes(KeepDeletedCells.TRUE);
s.close();
// Flush
r.flush(true);
// Major compact.
r.compact(true);
s = r.getScanner(new Scan());
int counter = 0;
do {
List<Cell> results = new ArrayList<>();
boolean result = s.next(results);
if (!result) break;
counter++;
} while(true);
assertEquals(0, counter);
} }
/** /**
@ -298,7 +283,7 @@ public class TestMajorCompaction {
final int ttl = 1000; final int ttl = 1000;
for (HStore store : r.getStores()) { for (HStore store : r.getStores()) {
ScanInfo old = store.getScanInfo(); ScanInfo old = store.getScanInfo();
ScanInfo si = old.customize(old.getMaxVersions(), ttl); ScanInfo si = old.customize(old.getMaxVersions(), ttl, old.getKeepDeletedCells());
store.setScanInfo(si); store.setScanInfo(si);
} }
Thread.sleep(1000); Thread.sleep(1000);
@ -471,7 +456,9 @@ public class TestMajorCompaction {
boolean result = s.next(results); boolean result = s.next(results);
assertTrue(!results.isEmpty()); assertTrue(!results.isEmpty());
r.delete(new Delete(CellUtil.cloneRow(results.get(0)))); r.delete(new Delete(CellUtil.cloneRow(results.get(0))));
if (!result) break; if (!result) {
break;
}
} while (true); } while (true);
s.close(); s.close();
// Flush // Flush
@ -485,10 +472,51 @@ public class TestMajorCompaction {
do { do {
List<Cell> results = new ArrayList<>(); List<Cell> results = new ArrayList<>();
boolean result = s.next(results); boolean result = s.next(results);
if (!result) break; if (!result) {
break;
}
counter++; counter++;
} while (true); } while (true);
s.close(); s.close();
assertEquals(0, counter); assertEquals(0, counter);
} }
private void testMajorCompactingWithDeletes(KeepDeletedCells keepDeletedCells)
throws IOException {
createStoreFile(r);
for (int i = 0; i < compactionThreshold; i++) {
createStoreFile(r);
}
// Now delete everything.
InternalScanner s = r.getScanner(new Scan());
int originalCount = 0;
do {
List<Cell> results = new ArrayList<>();
boolean result = s.next(results);
r.delete(new Delete(CellUtil.cloneRow(results.get(0))));
if (!result) break;
originalCount++;
} while (true);
s.close();
// Flush
r.flush(true);
for (HStore store : this.r.stores.values()) {
ScanInfo old = store.getScanInfo();
ScanInfo si = old.customize(old.getMaxVersions(), old.getTtl(), keepDeletedCells);
store.setScanInfo(si);
}
// Major compact.
r.compact(true);
s = r.getScanner(new Scan().setRaw(true));
int counter = 0;
do {
List<Cell> results = new ArrayList<>();
boolean result = s.next(results);
if (!result) break;
counter++;
} while (true);
assertEquals(keepDeletedCells == KeepDeletedCells.TRUE ? originalCount : 0, counter);
}
} }