Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-14761 Deletes with and without visibility expression do not delete 

the matching mutation (Ram)
This commit is contained in:
ramkrishna 2015-11-19 11:20:45 +05:30
parent 40e01512cf
commit b06acfbf0b
2 changed files with 377 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

166
hbase-server/src/main/java/org/apache/hadoop/hbase/security/visibility/VisibilityScanDeleteTracker.java
View File

@ -117,21 +117,26 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
// If tag is present in the delete // If tag is present in the delete
boolean hasVisTag = false; boolean hasVisTag = false;
if (delCell.getTagsLength() > 0) { if (delCell.getTagsLength() > 0) {
Byte deleteCellVisTagsFormat = null;
switch (type) { switch (type) {
case DeleteFamily: case DeleteFamily:
List<Tag> delTags = new ArrayList<Tag>(); List<Tag> delTags = new ArrayList<Tag>();
if (visibilityTagsDeleteFamily != null) { if (visibilityTagsDeleteFamily == null) {
Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags); visibilityTagsDeleteFamily = new ArrayList<Triple<List<Tag>, Byte, Long>>();
if (!delTags.isEmpty()) { }
visibilityTagsDeleteFamily.add(new Triple<List<Tag>, Byte, Long>( deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
delTags, deleteCellVisTagsFormat, delCell.getTimestamp())); if (!delTags.isEmpty()) {
hasVisTag = true; visibilityTagsDeleteFamily.add(new Triple<List<Tag>, Byte, Long>(delTags,
} deleteCellVisTagsFormat, delCell.getTimestamp()));
hasVisTag = true;
} }
break; break;
case DeleteFamilyVersion: case DeleteFamilyVersion:
if(visibilityTagsDeleteFamilyVersion == null) {
visibilityTagsDeleteFamilyVersion = new ArrayList<Triple<List<Tag>, Byte, Long>>();
}
delTags = new ArrayList<Tag>(); delTags = new ArrayList<Tag>();
Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags); deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
if (!delTags.isEmpty()) { if (!delTags.isEmpty()) {
visibilityTagsDeleteFamilyVersion.add(new Triple<List<Tag>, Byte, Long>(delTags, visibilityTagsDeleteFamilyVersion.add(new Triple<List<Tag>, Byte, Long>(delTags,
deleteCellVisTagsFormat, delCell.getTimestamp())); deleteCellVisTagsFormat, delCell.getTimestamp()));
@ -165,23 +170,6 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
default: default:
throw new IllegalArgumentException("Invalid delete type"); throw new IllegalArgumentException("Invalid delete type");
} }
} else {
switch (type) {
case DeleteFamily:
visibilityTagsDeleteFamily = null;
break;
case DeleteFamilyVersion:
visibilityTagsDeleteFamilyVersion = null;
break;
case DeleteColumn:
visibilityTagsDeleteColumns = null;
break;
case Delete:
visiblityTagsDeleteColumnVersion = null;
break;
default:
throw new IllegalArgumentException("Invalid delete type");
}
} }
return hasVisTag; return hasVisTag;
} }
@ -194,28 +182,35 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
try { try {
if (hasFamilyStamp) { if (hasFamilyStamp) {
if (visibilityTagsDeleteFamily != null) { if (visibilityTagsDeleteFamily != null) {
for (int i = 0; i < visibilityTagsDeleteFamily.size(); i++) { if (!visibilityTagsDeleteFamily.isEmpty()) {
// visibilityTagsDeleteFamily is ArrayList for (int i = 0; i < visibilityTagsDeleteFamily.size(); i++) {
Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamily.get(i); // visibilityTagsDeleteFamily is ArrayList
if (timestamp <= triple.getThird()) { Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamily.get(i);
List<Tag> putVisTags = new ArrayList<Tag>(); if (timestamp <= triple.getThird()) {
Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags); List<Tag> putVisTags = new ArrayList<Tag>();
boolean matchFound = VisibilityLabelServiceManager Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
.getInstance() boolean matchFound = VisibilityLabelServiceManager.getInstance()
.getVisibilityLabelService() .getVisibilityLabelService().matchVisibility(putVisTags, putCellVisTagsFormat,
.matchVisibility(putVisTags, putCellVisTagsFormat, triple.getFirst(), triple.getFirst(), triple.getSecond());
triple.getSecond()); if (matchFound) {
if (matchFound) { // A return type of FAMILY_DELETED will cause skip for all remaining cells from
// A return type of FAMILY_DELETED will cause skip for all remaining cells from this // this
// family. We would like to match visibility expression on every put cells after // family. We would like to match visibility expression on every put cells after
// this and only remove those matching with the family delete visibility. So we are // this and only remove those matching with the family delete visibility. So we
// returning FAMILY_VERSION_DELETED from here. // are
return DeleteResult.FAMILY_VERSION_DELETED; // returning FAMILY_VERSION_DELETED from here.
return DeleteResult.FAMILY_VERSION_DELETED;
}
} }
} }
} else {
if (!VisibilityUtils.isVisibilityTagsPresent(cell) && timestamp <= familyStamp) {
// No tags
return DeleteResult.FAMILY_VERSION_DELETED;
}
} }
} else { } else {
if (!VisibilityUtils.isVisibilityTagsPresent(cell) && timestamp<=familyStamp) { if (!VisibilityUtils.isVisibilityTagsPresent(cell) && timestamp <= familyStamp) {
// No tags // No tags
return DeleteResult.FAMILY_VERSION_DELETED; return DeleteResult.FAMILY_VERSION_DELETED;
} }
@ -223,21 +218,26 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
} }
if (familyVersionStamps.contains(Long.valueOf(timestamp))) { if (familyVersionStamps.contains(Long.valueOf(timestamp))) {
if (visibilityTagsDeleteFamilyVersion != null) { if (visibilityTagsDeleteFamilyVersion != null) {
for (int i = 0; i < visibilityTagsDeleteFamilyVersion.size(); i++) { if (!visibilityTagsDeleteFamilyVersion.isEmpty()) {
// visibilityTagsDeleteFamilyVersion is ArrayList for (int i = 0; i < visibilityTagsDeleteFamilyVersion.size(); i++) {
Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamilyVersion.get(i); // visibilityTagsDeleteFamilyVersion is ArrayList
if (timestamp == triple.getThird()) { Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamilyVersion.get(i);
List<Tag> putVisTags = new ArrayList<Tag>(); if (timestamp == triple.getThird()) {
Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags); List<Tag> putVisTags = new ArrayList<Tag>();
boolean matchFound = VisibilityLabelServiceManager Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
.getInstance() boolean matchFound = VisibilityLabelServiceManager.getInstance()
.getVisibilityLabelService() .getVisibilityLabelService().matchVisibility(putVisTags, putCellVisTagsFormat,
.matchVisibility(putVisTags, putCellVisTagsFormat, triple.getFirst(), triple.getFirst(), triple.getSecond());
triple.getSecond()); if (matchFound) {
if (matchFound) { return DeleteResult.FAMILY_VERSION_DELETED;
return DeleteResult.FAMILY_VERSION_DELETED; }
} }
} }
} else {
if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
// No tags
return DeleteResult.FAMILY_VERSION_DELETED;
}
} }
} else { } else {
if (!VisibilityUtils.isVisibilityTagsPresent(cell)) { if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
@ -253,15 +253,21 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
if (ret == 0) { if (ret == 0) {
if (deleteType == KeyValue.Type.DeleteColumn.getCode()) { if (deleteType == KeyValue.Type.DeleteColumn.getCode()) {
if (visibilityTagsDeleteColumns != null) { if (visibilityTagsDeleteColumns != null) {
for (Pair<List<Tag>, Byte> tags : visibilityTagsDeleteColumns) { if (!visibilityTagsDeleteColumns.isEmpty()) {
List<Tag> putVisTags = new ArrayList<Tag>(); for (Pair<List<Tag>, Byte> tags : visibilityTagsDeleteColumns) {
Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags); List<Tag> putVisTags = new ArrayList<Tag>();
boolean matchFound = VisibilityLabelServiceManager Byte putCellVisTagsFormat =
.getInstance() VisibilityUtils.extractVisibilityTags(cell, putVisTags);
.getVisibilityLabelService() boolean matchFound = VisibilityLabelServiceManager.getInstance()
.matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(), .getVisibilityLabelService().matchVisibility(putVisTags, putCellVisTagsFormat,
tags.getSecond()); tags.getFirst(), tags.getSecond());
if (matchFound) { if (matchFound) {
return DeleteResult.VERSION_DELETED;
}
}
} else {
if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
// No tags
return DeleteResult.VERSION_DELETED; return DeleteResult.VERSION_DELETED;
} }
} }
@ -276,15 +282,21 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
// If the timestamp is the same, keep this one // If the timestamp is the same, keep this one
if (timestamp == deleteTimestamp) { if (timestamp == deleteTimestamp) {
if (visiblityTagsDeleteColumnVersion != null) { if (visiblityTagsDeleteColumnVersion != null) {
for (Pair<List<Tag>, Byte> tags : visiblityTagsDeleteColumnVersion) { if (!visiblityTagsDeleteColumnVersion.isEmpty()) {
List<Tag> putVisTags = new ArrayList<Tag>(); for (Pair<List<Tag>, Byte> tags : visiblityTagsDeleteColumnVersion) {
Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags); List<Tag> putVisTags = new ArrayList<Tag>();
boolean matchFound = VisibilityLabelServiceManager Byte putCellVisTagsFormat =
.getInstance() VisibilityUtils.extractVisibilityTags(cell, putVisTags);
.getVisibilityLabelService() boolean matchFound = VisibilityLabelServiceManager.getInstance()
.matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(), .getVisibilityLabelService().matchVisibility(putVisTags, putCellVisTagsFormat,
tags.getSecond()); tags.getFirst(), tags.getSecond());
if (matchFound) { if (matchFound) {
return DeleteResult.VERSION_DELETED;
}
}
} else {
if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
// No tags
return DeleteResult.VERSION_DELETED; return DeleteResult.VERSION_DELETED;
} }
} }
@ -298,6 +310,7 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
} else if (ret < 0) { } else if (ret < 0) {
// Next column case. // Next column case.
deleteBuffer = null; deleteBuffer = null;
// Can nullify this because we are moving to the next column
visibilityTagsDeleteColumns = null; visibilityTagsDeleteColumns = null;
visiblityTagsDeleteColumnVersion = null; visiblityTagsDeleteColumnVersion = null;
} else { } else {
@ -316,9 +329,10 @@ public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
@Override @Override
public void reset() { public void reset() {
super.reset(); super.reset();
// clear only here
visibilityTagsDeleteColumns = null; visibilityTagsDeleteColumns = null;
visibilityTagsDeleteFamily = new ArrayList<Triple<List<Tag>, Byte, Long>>(); visibilityTagsDeleteFamily = null;
visibilityTagsDeleteFamilyVersion = new ArrayList<Triple<List<Tag>, Byte, Long>>(); visibilityTagsDeleteFamilyVersion = null;
visiblityTagsDeleteColumnVersion = null; visiblityTagsDeleteColumnVersion = null;
} }
} }

287
hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDeletes.java
View File

@ -469,6 +469,293 @@ public class TestVisibilityLabelsWithDeletes {
} }
} }
@Test
public void testDeleteColumnsWithoutAndWithVisibilityLabels() throws Exception {
final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
Admin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
HColumnDescriptor colDesc = new HColumnDescriptor(fam);
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(colDesc);
hBaseAdmin.createTable(desc);
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
Put put = new Put(row1);
put.addColumn(fam, qual, value);
put.setCellVisibility(new CellVisibility(CONFIDENTIAL));
table.put(put);
Delete d = new Delete(row1);
// without visibility
d.addColumns(fam, qual, HConstants.LATEST_TIMESTAMP);
table.delete(d);
PrivilegedExceptionAction<Void> scanAction = new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
assertEquals(next.length, 1);
} catch (Throwable t) {
throw new IOException(t);
}
return null;
}
};
SUPERUSER.runAs(scanAction);
d = new Delete(row1);
// with visibility
d.setCellVisibility(new CellVisibility(CONFIDENTIAL));
d.addColumns(fam, qual, HConstants.LATEST_TIMESTAMP);
table.delete(d);
scanAction = new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
assertEquals(next.length, 0);
} catch (Throwable t) {
throw new IOException(t);
}
return null;
}
};
SUPERUSER.runAs(scanAction);
}
}
@Test
public void testDeleteColumnsWithAndWithoutVisibilityLabels() throws Exception {
final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
Admin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
HColumnDescriptor colDesc = new HColumnDescriptor(fam);
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(colDesc);
hBaseAdmin.createTable(desc);
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
Put put = new Put(row1);
put.addColumn(fam, qual, value);
put.setCellVisibility(new CellVisibility(CONFIDENTIAL));
table.put(put);
Delete d = new Delete(row1);
// with visibility
d.setCellVisibility(new CellVisibility(CONFIDENTIAL));
d.addColumns(fam, qual, HConstants.LATEST_TIMESTAMP);
table.delete(d);
PrivilegedExceptionAction<Void> scanAction = new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
assertEquals(next.length, 0);
} catch (Throwable t) {
throw new IOException(t);
}
return null;
}
};
SUPERUSER.runAs(scanAction);
d = new Delete(row1);
// without visibility
d.addColumns(fam, qual, HConstants.LATEST_TIMESTAMP);
table.delete(d);
scanAction = new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
assertEquals(next.length, 0);
} catch (Throwable t) {
throw new IOException(t);
}
return null;
}
};
SUPERUSER.runAs(scanAction);
}
}
@Test
public void testDeleteFamiliesWithoutAndWithVisibilityLabels() throws Exception {
final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
Admin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
HColumnDescriptor colDesc = new HColumnDescriptor(fam);
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(colDesc);
hBaseAdmin.createTable(desc);
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
Put put = new Put(row1);
put.addColumn(fam, qual, value);
put.setCellVisibility(new CellVisibility(CONFIDENTIAL));
table.put(put);
Delete d = new Delete(row1);
// without visibility
d.addFamily(fam);
table.delete(d);
PrivilegedExceptionAction<Void> scanAction = new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
assertEquals(next.length, 1);
} catch (Throwable t) {
throw new IOException(t);
}
return null;
}
};
SUPERUSER.runAs(scanAction);
d = new Delete(row1);
// with visibility
d.setCellVisibility(new CellVisibility(CONFIDENTIAL));
d.addFamily(fam);
table.delete(d);
scanAction = new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
assertEquals(next.length, 0);
} catch (Throwable t) {
throw new IOException(t);
}
return null;
}
};
SUPERUSER.runAs(scanAction);
}
}
@Test
public void testDeleteFamiliesWithAndWithoutVisibilityLabels() throws Exception {
final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
Admin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
HColumnDescriptor colDesc = new HColumnDescriptor(fam);
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(colDesc);
hBaseAdmin.createTable(desc);
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
Put put = new Put(row1);
put.addColumn(fam, qual, value);
put.setCellVisibility(new CellVisibility(CONFIDENTIAL));
table.put(put);
Delete d = new Delete(row1);
d.setCellVisibility(new CellVisibility(CONFIDENTIAL));
// with visibility
d.addFamily(fam);
table.delete(d);
PrivilegedExceptionAction<Void> scanAction = new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
assertEquals(next.length, 0);
} catch (Throwable t) {
throw new IOException(t);
}
return null;
}
};
SUPERUSER.runAs(scanAction);
d = new Delete(row1);
// without visibility
d.addFamily(fam);
table.delete(d);
scanAction = new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
assertEquals(next.length, 0);
} catch (Throwable t) {
throw new IOException(t);
}
return null;
}
};
SUPERUSER.runAs(scanAction);
}
}
@Test
public void testDeletesWithoutAndWithVisibilityLabels() throws Exception {
final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
Admin hBaseAdmin = TEST_UTIL.getHBaseAdmin();
HColumnDescriptor colDesc = new HColumnDescriptor(fam);
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(colDesc);
hBaseAdmin.createTable(desc);
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
Put put = new Put(row1);
put.addColumn(fam, qual, value);
put.setCellVisibility(new CellVisibility(CONFIDENTIAL));
table.put(put);
Delete d = new Delete(row1);
// without visibility
d.addColumn(fam, qual);
table.delete(d);
PrivilegedExceptionAction<Void> scanAction = new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
// The delete would not be able to apply it because of visibility mismatch
Result[] next = scanner.next(3);
assertEquals(next.length, 1);
} catch (Throwable t) {
throw new IOException(t);
}
return null;
}
};
SUPERUSER.runAs(scanAction);
d = new Delete(row1);
// with visibility
d.setCellVisibility(new CellVisibility(CONFIDENTIAL));
d.addColumn(fam, qual);
table.delete(d);
scanAction = new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
try (Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName)) {
Scan s = new Scan();
ResultScanner scanner = table.getScanner(s);
Result[] next = scanner.next(3);
// this will alone match
assertEquals(next.length, 0);
} catch (Throwable t) {
throw new IOException(t);
}
return null;
}
};
SUPERUSER.runAs(scanAction);
}
}
@Test @Test
public void testVisibilityLabelsWithDeleteFamilyWithPutsReAppearing() throws Exception { public void testVisibilityLabelsWithDeleteFamilyWithPutsReAppearing() throws Exception {
final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());