HBASE-23711 - Add test for MinVersions and KeepDeletedCells TTL (#1079)
Signed-off-by: stack <stack@apache.org> Signed-off-by: Viraj Jasani <vjasani@apache.org>
This commit is contained in:
parent
2ed81c645b
commit
bb56dfa974
|
@ -27,17 +27,23 @@ import org.apache.hadoop.hbase.Cell;
|
|||
import org.apache.hadoop.hbase.CellUtil;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||
import org.apache.hadoop.hbase.KeepDeletedCells;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
|
||||
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.client.Delete;
|
||||
import org.apache.hadoop.hbase.client.Get;
|
||||
import org.apache.hadoop.hbase.client.Put;
|
||||
import org.apache.hadoop.hbase.client.Result;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptor;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.filter.TimestampsFilter;
|
||||
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
|
||||
import org.apache.hadoop.hbase.testclassification.SmallTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
|
||||
import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
|
||||
import org.junit.Assert;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
|
@ -54,7 +60,7 @@ public class TestMinVersions {
|
|||
public static final HBaseClassTestRule CLASS_RULE =
|
||||
HBaseClassTestRule.forClass(TestMinVersions.class);
|
||||
|
||||
HBaseTestingUtility hbu = HBaseTestingUtility.createLocalHTU();
|
||||
HBaseTestingUtility hbu = new HBaseTestingUtility();
|
||||
private final byte[] T0 = Bytes.toBytes("0");
|
||||
private final byte[] T1 = Bytes.toBytes("1");
|
||||
private final byte[] T2 = Bytes.toBytes("2");
|
||||
|
@ -71,8 +77,14 @@ public class TestMinVersions {
|
|||
*/
|
||||
@Test
|
||||
public void testGetClosestBefore() throws Exception {
|
||||
HTableDescriptor htd =
|
||||
hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, KeepDeletedCells.FALSE);
|
||||
|
||||
ColumnFamilyDescriptor cfd =
|
||||
ColumnFamilyDescriptorBuilder.newBuilder(c0)
|
||||
.setMinVersions(1).setMaxVersions(1000).setTimeToLive(1).
|
||||
setKeepDeletedCells(KeepDeletedCells.FALSE).build();
|
||||
|
||||
TableDescriptor htd = TableDescriptorBuilder.
|
||||
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
|
||||
HRegion region = hbu.createLocalHRegion(htd, null, null);
|
||||
try {
|
||||
|
||||
|
@ -121,8 +133,15 @@ public class TestMinVersions {
|
|||
@Test
|
||||
public void testStoreMemStore() throws Exception {
|
||||
// keep 3 versions minimum
|
||||
HTableDescriptor htd =
|
||||
hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE);
|
||||
|
||||
ColumnFamilyDescriptor cfd =
|
||||
ColumnFamilyDescriptorBuilder.newBuilder(c0)
|
||||
.setMinVersions(3).setMaxVersions(1000).setTimeToLive(1).
|
||||
setKeepDeletedCells(KeepDeletedCells.FALSE).build();
|
||||
|
||||
TableDescriptor htd = TableDescriptorBuilder.
|
||||
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
|
||||
|
||||
HRegion region = hbu.createLocalHRegion(htd, null, null);
|
||||
// 2s in the past
|
||||
long ts = EnvironmentEdgeManager.currentTime() - 2000;
|
||||
|
@ -176,8 +195,14 @@ public class TestMinVersions {
|
|||
*/
|
||||
@Test
|
||||
public void testDelete() throws Exception {
|
||||
HTableDescriptor htd =
|
||||
hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE);
|
||||
ColumnFamilyDescriptor cfd =
|
||||
ColumnFamilyDescriptorBuilder.newBuilder(c0)
|
||||
.setMinVersions(3).setMaxVersions(1000).setTimeToLive(1).
|
||||
setKeepDeletedCells(KeepDeletedCells.FALSE).build();
|
||||
|
||||
TableDescriptor htd = TableDescriptorBuilder.
|
||||
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
|
||||
|
||||
HRegion region = hbu.createLocalHRegion(htd, null, null);
|
||||
|
||||
// 2s in the past
|
||||
|
@ -235,8 +260,13 @@ public class TestMinVersions {
|
|||
*/
|
||||
@Test
|
||||
public void testMemStore() throws Exception {
|
||||
HTableDescriptor htd =
|
||||
hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
|
||||
ColumnFamilyDescriptor cfd =
|
||||
ColumnFamilyDescriptorBuilder.newBuilder(c0)
|
||||
.setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
|
||||
setKeepDeletedCells(KeepDeletedCells.FALSE).build();
|
||||
|
||||
TableDescriptor htd = TableDescriptorBuilder.
|
||||
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
|
||||
HRegion region = hbu.createLocalHRegion(htd, null, null);
|
||||
|
||||
// 2s in the past
|
||||
|
@ -310,9 +340,14 @@ public class TestMinVersions {
|
|||
*/
|
||||
@Test
|
||||
public void testBaseCase() throws Exception {
|
||||
// 1 version minimum, 1000 versions maximum, ttl = 1s
|
||||
HTableDescriptor htd =
|
||||
hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
|
||||
// 2 version minimum, 1000 versions maximum, ttl = 1s
|
||||
ColumnFamilyDescriptor cfd =
|
||||
ColumnFamilyDescriptorBuilder.newBuilder(c0)
|
||||
.setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
|
||||
setKeepDeletedCells(KeepDeletedCells.FALSE).build();
|
||||
|
||||
TableDescriptor htd = TableDescriptorBuilder.
|
||||
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
|
||||
HRegion region = hbu.createLocalHRegion(htd, null, null);
|
||||
try {
|
||||
|
||||
|
@ -403,10 +438,23 @@ public class TestMinVersions {
|
|||
*/
|
||||
@Test
|
||||
public void testFilters() throws Exception {
|
||||
HTableDescriptor htd =
|
||||
hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
|
||||
HRegion region = hbu.createLocalHRegion(htd, null, null);
|
||||
final byte [] c1 = COLUMNS[1];
|
||||
ColumnFamilyDescriptor cfd =
|
||||
ColumnFamilyDescriptorBuilder.newBuilder(c0)
|
||||
.setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
|
||||
setKeepDeletedCells(KeepDeletedCells.FALSE).build();
|
||||
|
||||
ColumnFamilyDescriptor cfd2 =
|
||||
ColumnFamilyDescriptorBuilder.newBuilder(c1)
|
||||
.setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
|
||||
setKeepDeletedCells(KeepDeletedCells.FALSE).build();
|
||||
List<ColumnFamilyDescriptor> cfdList = new ArrayList();
|
||||
cfdList.add(cfd);
|
||||
cfdList.add(cfd2);
|
||||
|
||||
TableDescriptor htd = TableDescriptorBuilder.
|
||||
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamilies(cfdList).build();
|
||||
HRegion region = hbu.createLocalHRegion(htd, null, null);
|
||||
|
||||
// 2s in the past
|
||||
long ts = EnvironmentEdgeManager.currentTime() - 2000;
|
||||
|
@ -476,12 +524,96 @@ public class TestMinVersions {
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMinVersionsWithKeepDeletedCellsTTL() throws Exception {
|
||||
int ttl = 4;
|
||||
ColumnFamilyDescriptor cfd =
|
||||
ColumnFamilyDescriptorBuilder.newBuilder(c0)
|
||||
.setMinVersions(2).setMaxVersions(Integer.MAX_VALUE).setTimeToLive(ttl).
|
||||
setKeepDeletedCells(KeepDeletedCells.TTL).build();
|
||||
|
||||
TableDescriptor htd = TableDescriptorBuilder.
|
||||
newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
|
||||
|
||||
HRegion region = hbu.createLocalHRegion(htd, null, null);
|
||||
|
||||
long startTS = EnvironmentEdgeManager.currentTime();
|
||||
ManualEnvironmentEdge injectEdge = new ManualEnvironmentEdge();
|
||||
injectEdge.setValue(startTS);
|
||||
EnvironmentEdgeManager.injectEdge(injectEdge);
|
||||
|
||||
long ts = startTS - 2000;
|
||||
// 1st version
|
||||
Put p = new Put(T1, ts-3);
|
||||
p.addColumn(c0, c0, T1);
|
||||
region.put(p);
|
||||
|
||||
// 2nd version
|
||||
p = new Put(T1, ts-2);
|
||||
p.addColumn(c0, c0, T2);
|
||||
region.put(p);
|
||||
|
||||
// 3rd version
|
||||
p = new Put(T1, ts-1);
|
||||
p.addColumn(c0, c0, T3);
|
||||
region.put(p);
|
||||
|
||||
Get g;
|
||||
Result r;
|
||||
|
||||
//check we can still see all versions before compaction
|
||||
g = new Get(T1);
|
||||
g.readAllVersions();
|
||||
g.setTimeRange(0, ts);
|
||||
r = region.get(g);
|
||||
checkResult(r, c0, T3, T2, T1);
|
||||
|
||||
region.flush(true);
|
||||
region.compact(true);
|
||||
Assert.assertEquals(startTS, EnvironmentEdgeManager.currentTime());
|
||||
long expiredTime = EnvironmentEdgeManager.currentTime() - ts - 3;
|
||||
Assert.assertTrue("TTL for T1 has expired", expiredTime < (ttl * 1000));
|
||||
//check that nothing was purged yet
|
||||
g = new Get(T1);
|
||||
g.readAllVersions();
|
||||
g.setTimeRange(0, ts);
|
||||
r = region.get(g);
|
||||
checkResult(r, c0, T3, T2, T1);
|
||||
|
||||
g = new Get(T1);
|
||||
g.readAllVersions();
|
||||
g.setTimeRange(0, ts -1);
|
||||
r = region.get(g);
|
||||
checkResult(r, c0, T2, T1);
|
||||
|
||||
injectEdge.incValue(ttl * 1000);
|
||||
|
||||
region.flush(true);
|
||||
region.compact(true);
|
||||
|
||||
//check that after compaction (which is after TTL) that only T1 was purged
|
||||
g = new Get(T1);
|
||||
g.readAllVersions();
|
||||
g.setTimeRange(0, ts);
|
||||
r = region.get(g);
|
||||
checkResult(r, c0, T3, T2);
|
||||
|
||||
g = new Get(T1);
|
||||
g.readAllVersions();
|
||||
g.setTimestamp(ts -2);
|
||||
r = region.get(g);
|
||||
checkResult(r, c0, T2);
|
||||
}
|
||||
|
||||
private void checkResult(Result r, byte[] col, byte[] ... vals) {
|
||||
assertEquals(r.size(), vals.length);
|
||||
assertEquals(vals.length, r.size());
|
||||
List<Cell> kvs = r.getColumnCells(col, col);
|
||||
assertEquals(kvs.size(), vals.length);
|
||||
for (int i=0;i<vals.length;i++) {
|
||||
assertTrue(CellUtil.matchingValue(kvs.get(i), vals[i]));
|
||||
String expected = Bytes.toString(vals[i]);
|
||||
String actual = Bytes.toString(CellUtil.cloneValue(kvs.get(i)));
|
||||
assertTrue(expected + " was expected but doesn't match " + actual,
|
||||
CellUtil.matchingValue(kvs.get(i), vals[i]));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue