HBASE-13734 Improper timestamp checking with VisibilityScanDeleteTracker.
This commit is contained in:
anoopsjohn
parent
f28e395290
commit
d45e0a7d41
|
|
@ -20,12 +20,7 @@ package org.apache.hadoop.hbase.security.visibility;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
|
@ -37,6 +32,7 @@ import org.apache.hadoop.hbase.Tag;
|
|||
import org.apache.hadoop.hbase.regionserver.ScanDeleteTracker;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.Pair;
|
||||
import org.apache.hadoop.hbase.util.Triple;
|
||||
|
||||
/**
|
||||
* Similar to ScanDeletTracker but tracks the visibility expression also before
|
||||
|
|
@ -55,12 +51,12 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
|
|||
// type would solve this problem and also ensure that the combination of different type
|
||||
// of deletes with diff ts would also work fine
|
||||
// Track per TS
|
||||
private Map<Long, Pair<List<Tag>, Byte>> visibilityTagsDeleteFamily =
|
||||
new HashMap<Long, Pair<List<Tag>, Byte>>();
|
||||
private List<Triple<List<Tag>, Byte, Long>> visibilityTagsDeleteFamily =
|
||||
new ArrayList<Triple<List<Tag>, Byte, Long>>();
|
||||
// Delete family version with different ts and different visibility expression could come.
|
||||
// Need to track it per ts.
|
||||
private Map<Long,Pair<List<Tag>, Byte>> visibilityTagsDeleteFamilyVersion =
|
||||
new HashMap<Long, Pair<List<Tag>, Byte>>();
|
||||
private List<Triple<List<Tag>, Byte, Long>> visibilityTagsDeleteFamilyVersion =
|
||||
new ArrayList<Triple<List<Tag>, Byte, Long>>();
|
||||
private List<Pair<List<Tag>, Byte>> visibilityTagsDeleteColumns;
|
||||
// Tracking as List<List> is to handle same ts cell but different visibility tag.
|
||||
// TODO : Need to handle puts with same ts but different vis tags.
|
||||
|
|
@ -80,8 +76,10 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
|
|||
byte type = delCell.getTypeByte();
|
||||
if (type == KeyValue.Type.DeleteFamily.getCode()) {
|
||||
hasFamilyStamp = true;
|
||||
//familyStamps.add(delCell.getTimestamp());
|
||||
extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamily);
|
||||
boolean hasVisTag = extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamily);
|
||||
if (!hasVisTag && timestamp > familyStamp) {
|
||||
familyStamp = timestamp;
|
||||
}
|
||||
return;
|
||||
} else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) {
|
||||
familyVersionStamps.add(timestamp);
|
||||
|
|
@ -115,8 +113,9 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
|
|||
extractDeleteCellVisTags(delCell, KeyValue.Type.codeToType(type));
|
||||
}
|
||||
|
||||
private void extractDeleteCellVisTags(Cell delCell, Type type) {
|
||||
private boolean extractDeleteCellVisTags(Cell delCell, Type type) {
|
||||
// If tag is present in the delete
|
||||
boolean hasVisTag = false;
|
||||
if (delCell.getTagsLength() > 0) {
|
||||
switch (type) {
|
||||
case DeleteFamily:
|
||||
|
|
@ -124,8 +123,9 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
|
|||
if (visibilityTagsDeleteFamily != null) {
|
||||
Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
|
||||
if (!delTags.isEmpty()) {
|
||||
visibilityTagsDeleteFamily.put(delCell.getTimestamp(), new Pair<List<Tag>, Byte>(
|
||||
delTags, deleteCellVisTagsFormat));
|
||||
visibilityTagsDeleteFamily.add(new Triple<List<Tag>, Byte, Long>(
|
||||
delTags, deleteCellVisTagsFormat, delCell.getTimestamp()));
|
||||
hasVisTag = true;
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
|
@ -133,8 +133,9 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
|
|||
delTags = new ArrayList<Tag>();
|
||||
Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
|
||||
if (!delTags.isEmpty()) {
|
||||
visibilityTagsDeleteFamilyVersion.put(delCell.getTimestamp(), new Pair<List<Tag>, Byte>(
|
||||
delTags, deleteCellVisTagsFormat));
|
||||
visibilityTagsDeleteFamilyVersion.add(new Triple<List<Tag>, Byte, Long>(delTags,
|
||||
deleteCellVisTagsFormat, delCell.getTimestamp()));
|
||||
hasVisTag = true;
|
||||
}
|
||||
break;
|
||||
case DeleteColumn:
|
||||
|
|
@ -146,6 +147,7 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
|
|||
if (!delTags.isEmpty()) {
|
||||
visibilityTagsDeleteColumns.add(new Pair<List<Tag>, Byte>(delTags,
|
||||
deleteCellVisTagsFormat));
|
||||
hasVisTag = true;
|
||||
}
|
||||
break;
|
||||
case Delete:
|
||||
|
|
@ -157,6 +159,7 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
|
|||
if (!delTags.isEmpty()) {
|
||||
visiblityTagsDeleteColumnVersion.add(new Pair<List<Tag>, Byte>(delTags,
|
||||
deleteCellVisTagsFormat));
|
||||
hasVisTag = true;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
|
|
@ -180,6 +183,7 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
|
|||
throw new IllegalArgumentException("Invalid delete type");
|
||||
}
|
||||
}
|
||||
return hasVisTag;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -190,26 +194,28 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
|
|||
try {
|
||||
if (hasFamilyStamp) {
|
||||
if (visibilityTagsDeleteFamily != null) {
|
||||
Set<Entry<Long, Pair<List<Tag>, Byte>>> deleteFamilies = visibilityTagsDeleteFamily
|
||||
.entrySet();
|
||||
Iterator<Entry<Long, Pair<List<Tag>, Byte>>> iterator = deleteFamilies.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
Entry<Long, Pair<List<Tag>, Byte>> entry = iterator.next();
|
||||
if (timestamp <= entry.getKey()) {
|
||||
for (int i = 0; i < visibilityTagsDeleteFamily.size(); i++) {
|
||||
// visibilityTagsDeleteFamily is ArrayList
|
||||
Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamily.get(i);
|
||||
if (timestamp <= triple.getThird()) {
|
||||
List<Tag> putVisTags = new ArrayList<Tag>();
|
||||
Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
|
||||
boolean matchFound = VisibilityLabelServiceManager
|
||||
.getInstance()
|
||||
.getVisibilityLabelService()
|
||||
.matchVisibility(putVisTags, putCellVisTagsFormat, entry.getValue().getFirst(),
|
||||
entry.getValue().getSecond());
|
||||
.matchVisibility(putVisTags, putCellVisTagsFormat, triple.getFirst(),
|
||||
triple.getSecond());
|
||||
if (matchFound) {
|
||||
// A return type of FAMILY_DELETED will cause skip for all remaining cells from this
|
||||
// family. We would like to match visibility expression on every put cells after
|
||||
// this and only remove those matching with the family delete visibility. So we are
|
||||
// returning FAMILY_VERSION_DELETED from here.
|
||||
return DeleteResult.FAMILY_VERSION_DELETED;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
|
||||
if (!VisibilityUtils.isVisibilityTagsPresent(cell) && timestamp<=familyStamp) {
|
||||
// No tags
|
||||
return DeleteResult.FAMILY_VERSION_DELETED;
|
||||
}
|
||||
|
|
@ -217,18 +223,20 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
|
|||
}
|
||||
if (familyVersionStamps.contains(Long.valueOf(timestamp))) {
|
||||
if (visibilityTagsDeleteFamilyVersion != null) {
|
||||
Pair<List<Tag>, Byte> tags = visibilityTagsDeleteFamilyVersion.get(Long
|
||||
.valueOf(timestamp));
|
||||
if (tags != null) {
|
||||
List<Tag> putVisTags = new ArrayList<Tag>();
|
||||
Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
|
||||
boolean matchFound = VisibilityLabelServiceManager
|
||||
.getInstance()
|
||||
.getVisibilityLabelService()
|
||||
.matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(),
|
||||
tags.getSecond());
|
||||
if (matchFound) {
|
||||
return DeleteResult.FAMILY_VERSION_DELETED;
|
||||
for (int i = 0; i < visibilityTagsDeleteFamilyVersion.size(); i++) {
|
||||
// visibilityTagsDeleteFamilyVersion is ArrayList
|
||||
Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamilyVersion.get(i);
|
||||
if (timestamp == triple.getThird()) {
|
||||
List<Tag> putVisTags = new ArrayList<Tag>();
|
||||
Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
|
||||
boolean matchFound = VisibilityLabelServiceManager
|
||||
.getInstance()
|
||||
.getVisibilityLabelService()
|
||||
.matchVisibility(putVisTags, putCellVisTagsFormat, triple.getFirst(),
|
||||
triple.getSecond());
|
||||
if (matchFound) {
|
||||
return DeleteResult.FAMILY_VERSION_DELETED;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
|
@ -309,8 +317,8 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
|
|||
public void reset() {
|
||||
super.reset();
|
||||
visibilityTagsDeleteColumns = null;
|
||||
visibilityTagsDeleteFamily = new HashMap<Long, Pair<List<Tag>, Byte>>();
|
||||
visibilityTagsDeleteFamilyVersion = new HashMap<Long, Pair<List<Tag>, Byte>>();
|
||||
visibilityTagsDeleteFamily = new ArrayList<Triple<List<Tag>, Byte, Long>>();
|
||||
visibilityTagsDeleteFamilyVersion = new ArrayList<Triple<List<Tag>, Byte, Long>>();
|
||||
visiblityTagsDeleteColumnVersion = null;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.client.Admin;
|
|||
import org.apache.hadoop.hbase.client.Connection;
|
||||
import org.apache.hadoop.hbase.client.ConnectionFactory;
|
||||
import org.apache.hadoop.hbase.client.Delete;
|
||||
import org.apache.hadoop.hbase.client.Get;
|
||||
import org.apache.hadoop.hbase.client.Put;
|
||||
import org.apache.hadoop.hbase.client.Result;
|
||||
import org.apache.hadoop.hbase.client.ResultScanner;
|
||||
|
|
@ -2827,6 +2828,111 @@ public class TestVisibilityLabelsWithDeletes {
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeleteWithNoVisibilitiesForPutsAndDeletes() throws Exception {
|
||||
final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
|
||||
Admin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
|
||||
HColumnDescriptor colDesc = new HColumnDescriptor(fam);
|
||||
colDesc.setMaxVersions(5);
|
||||
HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||
desc.addFamily(colDesc);
|
||||
hBaseAdmin.createTable(desc);
|
||||
Put p = new Put (Bytes.toBytes("row1"));
|
||||
p.addColumn(fam, qual, value);
|
||||
Table table = TEST_UTIL.getConnection().getTable(tableName);
|
||||
table.put(p);
|
||||
p = new Put (Bytes.toBytes("row1"));
|
||||
p.addColumn(fam, qual1, value);
|
||||
table.put(p);
|
||||
p = new Put (Bytes.toBytes("row2"));
|
||||
p.addColumn(fam, qual, value);
|
||||
table.put(p);
|
||||
p = new Put (Bytes.toBytes("row2"));
|
||||
p.addColumn(fam, qual1, value);
|
||||
table.put(p);
|
||||
Delete d = new Delete(Bytes.toBytes("row1"));
|
||||
table.delete(d);
|
||||
Get g = new Get(Bytes.toBytes("row1"));
|
||||
g.setMaxVersions();
|
||||
g.setAuthorizations(new Authorizations(SECRET, PRIVATE));
|
||||
Result result = table.get(g);
|
||||
assertEquals(0, result.rawCells().length);
|
||||
|
||||
p = new Put (Bytes.toBytes("row1"));
|
||||
p.addColumn(fam, qual, value);
|
||||
table.put(p);
|
||||
result = table.get(g);
|
||||
assertEquals(1, result.rawCells().length);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeleteWithFamilyDeletesOfSameTsButDifferentVisibilities() throws Exception {
|
||||
final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
|
||||
Admin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
|
||||
HColumnDescriptor colDesc = new HColumnDescriptor(fam);
|
||||
colDesc.setMaxVersions(5);
|
||||
HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||
desc.addFamily(colDesc);
|
||||
hBaseAdmin.createTable(desc);
|
||||
Table table = TEST_UTIL.getConnection().getTable(tableName);
|
||||
long t1 = 1234L;
|
||||
CellVisibility cellVisibility1 = new CellVisibility(SECRET);
|
||||
CellVisibility cellVisibility2 = new CellVisibility(PRIVATE);
|
||||
// Cell row1:info:qual:1234 with visibility SECRET
|
||||
Put p = new Put(row1);
|
||||
p.addColumn(fam, qual, t1, value);
|
||||
p.setCellVisibility(cellVisibility1);
|
||||
table.put(p);
|
||||
|
||||
// Cell row1:info:qual1:1234 with visibility PRIVATE
|
||||
p = new Put(row1);
|
||||
p.addColumn(fam, qual1, t1, value);
|
||||
p.setCellVisibility(cellVisibility2);
|
||||
table.put(p);
|
||||
|
||||
Delete d = new Delete(row1);
|
||||
d.addFamily(fam, t1);
|
||||
d.setCellVisibility(cellVisibility2);
|
||||
table.delete(d);
|
||||
d = new Delete(row1);
|
||||
d.addFamily(fam, t1);
|
||||
d.setCellVisibility(cellVisibility1);
|
||||
table.delete(d);
|
||||
|
||||
Get g = new Get(row1);
|
||||
g.setMaxVersions();
|
||||
g.setAuthorizations(new Authorizations(SECRET, PRIVATE));
|
||||
Result result = table.get(g);
|
||||
assertEquals(0, result.rawCells().length);
|
||||
|
||||
// Cell row2:info:qual:1234 with visibility SECRET
|
||||
p = new Put(row2);
|
||||
p.addColumn(fam, qual, t1, value);
|
||||
p.setCellVisibility(cellVisibility1);
|
||||
table.put(p);
|
||||
|
||||
// Cell row2:info:qual1:1234 with visibility PRIVATE
|
||||
p = new Put(row2);
|
||||
p.addColumn(fam, qual1, t1, value);
|
||||
p.setCellVisibility(cellVisibility2);
|
||||
table.put(p);
|
||||
|
||||
d = new Delete(row2);
|
||||
d.addFamilyVersion(fam, t1);
|
||||
d.setCellVisibility(cellVisibility2);
|
||||
table.delete(d);
|
||||
d = new Delete(row2);
|
||||
d.addFamilyVersion(fam, t1);
|
||||
d.setCellVisibility(cellVisibility1);
|
||||
table.delete(d);
|
||||
|
||||
g = new Get(row2);
|
||||
g.setMaxVersions();
|
||||
g.setAuthorizations(new Authorizations(SECRET, PRIVATE));
|
||||
result = table.get(g);
|
||||
assertEquals(0, result.rawCells().length);
|
||||
}
|
||||
|
||||
public static Table createTableAndWriteDataWithLabels(TableName tableName, String... labelExps)
|
||||
throws Exception {
|
||||
Table table = null;
|
||||
|
|
|
|||
Loading…
Reference in New Issue