Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-13734 Improper timestamp checking with VisibilityScanDeleteTracker.

This commit is contained in:
anoopsjohn 2015-05-25 13:54:24 +05:30
parent 2542a38e21
commit fd8e6ee956
2 changed files with 153 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java
View File

@ -20,12 +20,7 @@ package org.apache.hadoop.hbase.security.visibility;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -37,6 +32,7 @@ import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.regionserver.ScanDeleteTracker;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Triple;
/**
* Similar to ScanDeletTracker but tracks the visibility expression also before
@ -55,12 +51,12 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
// type would solve this problem and also ensure that the combination of different type
// of deletes with diff ts would also work fine
// Track per TS
private Map<Long, Pair<List<Tag>, Byte>> visibilityTagsDeleteFamily =
new HashMap<Long, Pair<List<Tag>, Byte>>();
private List<Triple<List<Tag>, Byte, Long>> visibilityTagsDeleteFamily =
new ArrayList<Triple<List<Tag>, Byte, Long>>();
// Delete family version with different ts and different visibility expression could come.
// Need to track it per ts.
private Map<Long,Pair<List<Tag>, Byte>> visibilityTagsDeleteFamilyVersion =
new HashMap<Long, Pair<List<Tag>, Byte>>();
private List<Triple<List<Tag>, Byte, Long>> visibilityTagsDeleteFamilyVersion =
new ArrayList<Triple<List<Tag>, Byte, Long>>();
private List<Pair<List<Tag>, Byte>> visibilityTagsDeleteColumns;
// Tracking as List<List> is to handle same ts cell but different visibility tag.
// TODO : Need to handle puts with same ts but different vis tags.
@ -80,8 +76,10 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
byte type = delCell.getTypeByte();
if (type == KeyValue.Type.DeleteFamily.getCode()) {
hasFamilyStamp = true;
//familyStamps.add(delCell.getTimestamp());
extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamily);
boolean hasVisTag = extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamily);
if (!hasVisTag && timestamp > familyStamp) {
familyStamp = timestamp;
}
return;
} else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) {
familyVersionStamps.add(timestamp);
@ -115,8 +113,9 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
extractDeleteCellVisTags(delCell, KeyValue.Type.codeToType(type));
}
private void extractDeleteCellVisTags(Cell delCell, Type type) {
private boolean extractDeleteCellVisTags(Cell delCell, Type type) {
// If tag is present in the delete
boolean hasVisTag = false;
if (delCell.getTagsLength() > 0) {
switch (type) {
case DeleteFamily:
@ -124,8 +123,9 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
if (visibilityTagsDeleteFamily != null) {
Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
if (!delTags.isEmpty()) {
visibilityTagsDeleteFamily.put(delCell.getTimestamp(), new Pair<List<Tag>, Byte>(
delTags, deleteCellVisTagsFormat));
visibilityTagsDeleteFamily.add(new Triple<List<Tag>, Byte, Long>(
delTags, deleteCellVisTagsFormat, delCell.getTimestamp()));
hasVisTag = true;
}
}
break;
@ -133,8 +133,9 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
delTags = new ArrayList<Tag>();
Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
if (!delTags.isEmpty()) {
visibilityTagsDeleteFamilyVersion.put(delCell.getTimestamp(), new Pair<List<Tag>, Byte>(
delTags, deleteCellVisTagsFormat));
visibilityTagsDeleteFamilyVersion.add(new Triple<List<Tag>, Byte, Long>(delTags,
deleteCellVisTagsFormat, delCell.getTimestamp()));
hasVisTag = true;
}
break;
case DeleteColumn:
@ -146,6 +147,7 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
if (!delTags.isEmpty()) {
visibilityTagsDeleteColumns.add(new Pair<List<Tag>, Byte>(delTags,
deleteCellVisTagsFormat));
hasVisTag = true;
}
break;
case Delete:
@ -157,6 +159,7 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
if (!delTags.isEmpty()) {
visiblityTagsDeleteColumnVersion.add(new Pair<List<Tag>, Byte>(delTags,
deleteCellVisTagsFormat));
hasVisTag = true;
}
break;
default:
@ -180,6 +183,7 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
throw new IllegalArgumentException("Invalid delete type");
}
}
return hasVisTag;
}
@Override
@ -190,26 +194,28 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
try {
if (hasFamilyStamp) {
if (visibilityTagsDeleteFamily != null) {
Set<Entry<Long, Pair<List<Tag>, Byte>>> deleteFamilies = visibilityTagsDeleteFamily
.entrySet();
Iterator<Entry<Long, Pair<List<Tag>, Byte>>> iterator = deleteFamilies.iterator();
while (iterator.hasNext()) {
Entry<Long, Pair<List<Tag>, Byte>> entry = iterator.next();
if (timestamp <= entry.getKey()) {
for (int i = 0; i < visibilityTagsDeleteFamily.size(); i++) {
// visibilityTagsDeleteFamily is ArrayList
Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamily.get(i);
if (timestamp <= triple.getThird()) {
List<Tag> putVisTags = new ArrayList<Tag>();
Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
boolean matchFound = VisibilityLabelServiceManager
.getInstance()
.getVisibilityLabelService()
.matchVisibility(putVisTags, putCellVisTagsFormat, entry.getValue().getFirst(),
entry.getValue().getSecond());
.matchVisibility(putVisTags, putCellVisTagsFormat, triple.getFirst(),
triple.getSecond());
if (matchFound) {
// A return type of FAMILY_DELETED will cause skip for all remaining cells from this
// family. We would like to match visibility expression on every put cells after
// this and only remove those matching with the family delete visibility. So we are
// returning FAMILY_VERSION_DELETED from here.
return DeleteResult.FAMILY_VERSION_DELETED;
}
}
}
} else {
if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
if (!VisibilityUtils.isVisibilityTagsPresent(cell) && timestamp<=familyStamp) {
// No tags
return DeleteResult.FAMILY_VERSION_DELETED;
}
@ -217,18 +223,20 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
}
if (familyVersionStamps.contains(Long.valueOf(timestamp))) {
if (visibilityTagsDeleteFamilyVersion != null) {
Pair<List<Tag>, Byte> tags = visibilityTagsDeleteFamilyVersion.get(Long
.valueOf(timestamp));
if (tags != null) {
List<Tag> putVisTags = new ArrayList<Tag>();
Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
boolean matchFound = VisibilityLabelServiceManager
.getInstance()
.getVisibilityLabelService()
.matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(),
tags.getSecond());
if (matchFound) {
return DeleteResult.FAMILY_VERSION_DELETED;
for (int i = 0; i < visibilityTagsDeleteFamilyVersion.size(); i++) {
// visibilityTagsDeleteFamilyVersion is ArrayList
Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamilyVersion.get(i);
if (timestamp == triple.getThird()) {
List<Tag> putVisTags = new ArrayList<Tag>();
Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
boolean matchFound = VisibilityLabelServiceManager
.getInstance()
.getVisibilityLabelService()
.matchVisibility(putVisTags, putCellVisTagsFormat, triple.getFirst(),
triple.getSecond());
if (matchFound) {
return DeleteResult.FAMILY_VERSION_DELETED;
}
}
}
} else {
@ -309,8 +317,8 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
public void reset() {
super.reset();
visibilityTagsDeleteColumns = null;
visibilityTagsDeleteFamily = new HashMap<Long, Pair<List<Tag>, Byte>>();
visibilityTagsDeleteFamilyVersion = new HashMap<Long, Pair<List<Tag>, Byte>>();
visibilityTagsDeleteFamily = new ArrayList<Triple<List<Tag>, Byte, Long>>();
visibilityTagsDeleteFamilyVersion = new ArrayList<Triple<List<Tag>, Byte, Long>>();
visiblityTagsDeleteColumnVersion = null;
}
}

106
hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java
View File

@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
@ -2735,6 +2736,111 @@ public class TestVisibilityLabelsWithDeletes {
}
}
@Test
public void testDeleteWithNoVisibilitiesForPutsAndDeletes() throws Exception {
final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
Admin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
HColumnDescriptor colDesc = new HColumnDescriptor(fam);
colDesc.setMaxVersions(5);
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(colDesc);
hBaseAdmin.createTable(desc);
Put p = new Put (Bytes.toBytes("row1"));
p.addColumn(fam, qual, value);
Table table = TEST_UTIL.getConnection().getTable(tableName);
table.put(p);
p = new Put (Bytes.toBytes("row1"));
p.addColumn(fam, qual1, value);
table.put(p);
p = new Put (Bytes.toBytes("row2"));
p.addColumn(fam, qual, value);
table.put(p);
p = new Put (Bytes.toBytes("row2"));
p.addColumn(fam, qual1, value);
table.put(p);
Delete d = new Delete(Bytes.toBytes("row1"));
table.delete(d);
Get g = new Get(Bytes.toBytes("row1"));
g.setMaxVersions();
g.setAuthorizations(new Authorizations(SECRET, PRIVATE));
Result result = table.get(g);
assertEquals(0, result.rawCells().length);
p = new Put (Bytes.toBytes("row1"));
p.addColumn(fam, qual, value);
table.put(p);
result = table.get(g);
assertEquals(1, result.rawCells().length);
}
@Test
public void testDeleteWithFamilyDeletesOfSameTsButDifferentVisibilities() throws Exception {
final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
Admin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
HColumnDescriptor colDesc = new HColumnDescriptor(fam);
colDesc.setMaxVersions(5);
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(colDesc);
hBaseAdmin.createTable(desc);
Table table = TEST_UTIL.getConnection().getTable(tableName);
long t1 = 1234L;
CellVisibility cellVisibility1 = new CellVisibility(SECRET);
CellVisibility cellVisibility2 = new CellVisibility(PRIVATE);
// Cell row1:info:qual:1234 with visibility SECRET
Put p = new Put(row1);
p.addColumn(fam, qual, t1, value);
p.setCellVisibility(cellVisibility1);
table.put(p);
// Cell row1:info:qual1:1234 with visibility PRIVATE
p = new Put(row1);
p.addColumn(fam, qual1, t1, value);
p.setCellVisibility(cellVisibility2);
table.put(p);
Delete d = new Delete(row1);
d.addFamily(fam, t1);
d.setCellVisibility(cellVisibility2);
table.delete(d);
d = new Delete(row1);
d.addFamily(fam, t1);
d.setCellVisibility(cellVisibility1);
table.delete(d);
Get g = new Get(row1);
g.setMaxVersions();
g.setAuthorizations(new Authorizations(SECRET, PRIVATE));
Result result = table.get(g);
assertEquals(0, result.rawCells().length);
// Cell row2:info:qual:1234 with visibility SECRET
p = new Put(row2);
p.addColumn(fam, qual, t1, value);
p.setCellVisibility(cellVisibility1);
table.put(p);
// Cell row2:info:qual1:1234 with visibility PRIVATE
p = new Put(row2);
p.addColumn(fam, qual1, t1, value);
p.setCellVisibility(cellVisibility2);
table.put(p);
d = new Delete(row2);
d.addFamilyVersion(fam, t1);
d.setCellVisibility(cellVisibility2);
table.delete(d);
d = new Delete(row2);
d.addFamilyVersion(fam, t1);
d.setCellVisibility(cellVisibility1);
table.delete(d);
g = new Get(row2);
g.setMaxVersions();
g.setAuthorizations(new Authorizations(SECRET, PRIVATE));
result = table.get(g);
assertEquals(0, result.rawCells().length);
}
private void scanAll(Result[] next) throws IOException {
CellScanner cellScanner = next[0].cellScanner();
cellScanner.advance();