HBASE-19895 Add keepDeletedCells option in ScanOptions for customizing scanInfo in pre-hooks (Ankit Singhal)

This commit is contained in:
tedyu 2018-02-01 18:27:18 -08:00
parent 99b9fff07b
commit a11258599e
6 changed files with 87 additions and 38 deletions

View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.regionserver;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.yetus.audience.InterfaceAudience;
/**
@ -31,6 +32,8 @@ public class CustomizedScanInfoBuilder implements ScanOptions {
private Long ttl;
private KeepDeletedCells keepDeletedCells = null;
public CustomizedScanInfoBuilder(ScanInfo scanInfo) {
this.scanInfo = scanInfo;
}
@ -56,14 +59,25 @@ public class CustomizedScanInfoBuilder implements ScanOptions {
}
public ScanInfo build() {
if (maxVersions == null && ttl == null) {
if (maxVersions == null && ttl == null && keepDeletedCells == null) {
return scanInfo;
}
return scanInfo.customize(getMaxVersions(), getTTL());
return scanInfo.customize(getMaxVersions(), getTTL(), getKeepDeletedCells());
}
@Override
public String toString() {
return "ScanOptions [maxVersions=" + getMaxVersions() + ", TTL=" + getTTL() + "]";
}
@Override
public void setKeepDeletedCells(KeepDeletedCells keepDeletedCells) {
this.keepDeletedCells = keepDeletedCells;
}
@Override
public KeepDeletedCells getKeepDeletedCells() {
return keepDeletedCells != null ? keepDeletedCells : scanInfo.getKeepDeletedCells();
}
}

View File

@ -171,10 +171,11 @@ public class ScanInfo {
}
/**
* Used for CP users for customizing max versions and ttl.
* Used for CP users for customizing max versions, ttl and keepDeletedCells.
*/
ScanInfo customize(int maxVersions, long ttl) {
return new ScanInfo(family, minVersions, maxVersions, ttl, keepDeletedCells, ttl, comparator,
ttl, usePread, maxVersions, parallelSeekEnabled, ttl, newVersionBehavior);
ScanInfo customize(int maxVersions, long ttl, KeepDeletedCells keepDeletedCells) {
return new ScanInfo(family, minVersions, maxVersions, ttl, keepDeletedCells, timeToPurgeDeletes,
comparator, tableMaxRowSize, usePread, cellsPerTimeoutCheck, parallelSeekEnabled,
preadMaxBytes, newVersionBehavior);
}
}

View File

@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.regionserver;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
@ -59,4 +60,8 @@ public interface ScanOptions {
long getTTL();
void setTTL(long ttl);
void setKeepDeletedCells(KeepDeletedCells keepDeletedCells);
KeepDeletedCells getKeepDeletedCells();
}

View File

@ -218,7 +218,7 @@ public class TestCompaction {
final int ttl = 1000;
for (HStore store : this.r.stores.values()) {
ScanInfo old = store.getScanInfo();
ScanInfo si = old.customize(old.getMaxVersions(), ttl);
ScanInfo si = old.customize(old.getMaxVersions(), ttl, old.getKeepDeletedCells());
store.setScanInfo(si);
}
Thread.sleep(ttl);

View File

@ -170,7 +170,8 @@ public class TestDefaultCompactSelection extends TestCompactionPolicy {
public void testCompactionEmptyHFile() throws IOException {
// Set TTL
ScanInfo oldScanInfo = store.getScanInfo();
ScanInfo newScanInfo = oldScanInfo.customize(oldScanInfo.getMaxVersions(), 600);
ScanInfo newScanInfo = oldScanInfo.customize(oldScanInfo.getMaxVersions(), 600,
oldScanInfo.getKeepDeletedCells());
store.setScanInfo(newScanInfo);
// Do not compact empty store file
List<HStoreFile> candidates = sfCreate(0);

View File

@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
@ -134,36 +135,20 @@ public class TestMajorCompaction {
* Test that on a major compaction, if all cells are expired or deleted, then
* we'll end up with no product. Make sure scanner over region returns
* right answer in this case - and that it just basically works.
* @throws IOException
* @throws IOException exception encountered
*/
@Test
public void testMajorCompactingToNoOutput() throws IOException {
createStoreFile(r);
for (int i = 0; i < compactionThreshold; i++) {
createStoreFile(r);
}
// Now delete everything.
InternalScanner s = r.getScanner(new Scan());
do {
List<Cell> results = new ArrayList<>();
boolean result = s.next(results);
r.delete(new Delete(CellUtil.cloneRow(results.get(0))));
if (!result) break;
} while(true);
s.close();
// Flush
r.flush(true);
// Major compact.
r.compact(true);
s = r.getScanner(new Scan());
int counter = 0;
do {
List<Cell> results = new ArrayList<>();
boolean result = s.next(results);
if (!result) break;
counter++;
} while(true);
assertEquals(0, counter);
testMajorCompactingWithDeletes(KeepDeletedCells.FALSE);
}
/**
* Test that on a major compaction,Deleted cells are retained if keep deleted cells is set to true
* @throws IOException exception encountered
*/
@Test
public void testMajorCompactingWithKeepDeletedCells() throws IOException {
testMajorCompactingWithDeletes(KeepDeletedCells.TRUE);
}
/**
@ -298,7 +283,7 @@ public class TestMajorCompaction {
final int ttl = 1000;
for (HStore store : r.getStores()) {
ScanInfo old = store.getScanInfo();
ScanInfo si = old.customize(old.getMaxVersions(), ttl);
ScanInfo si = old.customize(old.getMaxVersions(), ttl, old.getKeepDeletedCells());
store.setScanInfo(si);
}
Thread.sleep(1000);
@ -471,7 +456,9 @@ public class TestMajorCompaction {
boolean result = s.next(results);
assertTrue(!results.isEmpty());
r.delete(new Delete(CellUtil.cloneRow(results.get(0))));
if (!result) break;
if (!result) {
break;
}
} while (true);
s.close();
// Flush
@ -485,10 +472,51 @@ public class TestMajorCompaction {
do {
List<Cell> results = new ArrayList<>();
boolean result = s.next(results);
if (!result) break;
if (!result) {
break;
}
counter++;
} while (true);
s.close();
assertEquals(0, counter);
}
private void testMajorCompactingWithDeletes(KeepDeletedCells keepDeletedCells)
throws IOException {
createStoreFile(r);
for (int i = 0; i < compactionThreshold; i++) {
createStoreFile(r);
}
// Now delete everything.
InternalScanner s = r.getScanner(new Scan());
int originalCount = 0;
do {
List<Cell> results = new ArrayList<>();
boolean result = s.next(results);
r.delete(new Delete(CellUtil.cloneRow(results.get(0))));
if (!result) break;
originalCount++;
} while (true);
s.close();
// Flush
r.flush(true);
for (HStore store : this.r.stores.values()) {
ScanInfo old = store.getScanInfo();
ScanInfo si = old.customize(old.getMaxVersions(), old.getTtl(), keepDeletedCells);
store.setScanInfo(si);
}
// Major compact.
r.compact(true);
s = r.getScanner(new Scan().setRaw(true));
int counter = 0;
do {
List<Cell> results = new ArrayList<>();
boolean result = s.next(results);
if (!result) break;
counter++;
} while (true);
assertEquals(keepDeletedCells == KeepDeletedCells.TRUE ? originalCount : 0, counter);
}
}