HBASE-22146 Removing a namespace-level space quota does not remove policies against contained tables

Closes #1935

Signed-off-by: Josh Elser <elserj@apache.org>
This commit is contained in:
surbhi 2020-06-19 16:21:22 -07:00 committed by Josh Elser
parent a6e3db5ba5
commit 8191fbdd7d
4 changed files with 157 additions and 1 deletions

View File

@ -628,6 +628,34 @@ public class QuotaTableUtil {
}
}
/**
* Remove table usage snapshots (u:p columns) for the namespace passed
* @param connection connection to re-use
* @param namespace the namespace to fetch the list of table usage snapshots
*/
static void deleteTableUsageSnapshotsForNamespace(Connection connection, String namespace)
throws IOException {
Scan s = new Scan();
//Get rows for all tables in namespace
s.setRowPrefixFilter(Bytes.add(QUOTA_TABLE_ROW_KEY_PREFIX, Bytes.toBytes(namespace + TableName.NAMESPACE_DELIM)));
//Scan for table usage column (u:p) in quota table
s.addColumn(QUOTA_FAMILY_USAGE,QUOTA_QUALIFIER_POLICY);
//Scan for table quota column (q:s) if table has a space quota defined
s.addColumn(QUOTA_FAMILY_INFO,QUOTA_QUALIFIER_SETTINGS);
try (Table quotaTable = connection.getTable(QUOTA_TABLE_NAME);
ResultScanner rs = quotaTable.getScanner(s)) {
for (Result r : rs) {
byte[] data = r.getValue(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS);
//if table does not have a table space quota defined, delete table usage column (u:p)
if (data == null) {
Delete delete = new Delete(r.getRow());
delete.addColumns(QUOTA_FAMILY_USAGE,QUOTA_QUALIFIER_POLICY);
quotaTable.delete(delete);
}
}
}
}
/**
* Fetches the computed size of all snapshots against tables in a namespace for space quotas.
*/

View File

@ -266,6 +266,14 @@ public class QuotaUtil extends QuotaTableUtil {
if (qualifier != null) {
delete.addColumns(QUOTA_FAMILY_INFO, qualifier);
}
if (isNamespaceRowKey(rowKey)) {
String ns = getNamespaceFromRowKey(rowKey);
Quotas namespaceQuota = getNamespaceQuota(connection,ns);
if (namespaceQuota != null && namespaceQuota.hasSpace()) {
// When deleting namespace space quota, also delete table usage(u:p) snapshots
deleteTableUsageSnapshotsForNamespace(connection, ns);
}
}
doDelete(connection, delete);
}

View File

@ -158,6 +158,28 @@ public class SpaceQuotaHelperForTests {
return tn;
}
TableName writeUntilViolationAndVerifyViolationInNamespace(
String ns, SpaceViolationPolicy policyToViolate, Mutation m) throws Exception {
final TableName tn = writeUntilViolationInNamespace(ns, policyToViolate);
verifyViolation(policyToViolate, tn, m);
return tn;
}
TableName writeUntilViolationInNamespace(String ns, SpaceViolationPolicy policyToViolate) throws Exception {
TableName tn = createTableWithRegions(ns,10);
setQuotaLimit(ns, policyToViolate, 4L);
// Write more data than should be allowed and flush it to disk
writeData(tn, 5L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
// This should be sufficient time for the chores to run and see the change.
Thread.sleep(5000);
return tn;
}
/**
* Verifies that the given policy on the given table has been violated
*/
@ -260,6 +282,19 @@ public class SpaceQuotaHelperForTests {
assertTrue("Expected to succeed in writing data to a table not having quota ", sawSuccess);
}
/**
* Verifies that table usage snapshot exists for the table
*/
void verifyTableUsageSnapshotForSpaceQuotaExist(TableName tn) throws Exception {
boolean sawUsageSnapshot = false;
try (Table quotaTable = testUtil.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
Scan s = QuotaTableUtil.makeQuotaSnapshotScanForTable(tn);
ResultScanner rs = quotaTable.getScanner(s);
sawUsageSnapshot = (rs.next() != null);
}
assertTrue("Expected to succeed in getting table usage snapshots for space quota", sawUsageSnapshot);
}
/**
* Sets the given quota (policy & limit) on the passed table.
*/
@ -271,6 +306,17 @@ public class SpaceQuotaHelperForTests {
LOG.debug("Quota limit set for table = {}, limit = {}", tn, sizeLimit);
}
/**
* Sets the given quota (policy & limit) on the passed namespace.
*/
void setQuotaLimit(String ns, SpaceViolationPolicy policy, long sizeInMBs)
throws Exception {
final long sizeLimit = sizeInMBs * SpaceQuotaHelperForTests.ONE_MEGABYTE;
QuotaSettings settings = QuotaSettingsFactory.limitNamespaceSpace(ns, sizeLimit, policy);
testUtil.getAdmin().setQuota(settings);
LOG.debug("Quota limit set for namespace = {}, limit = {}", ns, sizeLimit);
}
/**
* Removes the space quota from the given table
*/
@ -280,6 +326,16 @@ public class SpaceQuotaHelperForTests {
LOG.debug("Space quota settings removed from the table ", tn);
}
/**
* Removes the space quota from the given namespace
*/
void removeQuotaFromNamespace(String ns) throws Exception {
QuotaSettings removeQuota = QuotaSettingsFactory.removeNamespaceSpaceLimit(ns);
Admin admin = testUtil.getAdmin();
admin.setQuota(removeQuota);
LOG.debug("Space quota settings removed from the namespace ", ns);
}
/**
* Removes all quotas defined in the HBase quota table.
*/
@ -411,7 +467,13 @@ public class SpaceQuotaHelperForTests {
}
NamespaceDescriptor createNamespace() throws Exception {
NamespaceDescriptor nd = NamespaceDescriptor.create("ns" + counter.getAndIncrement()).build();
return createNamespace(null);
}
NamespaceDescriptor createNamespace(String namespace) throws Exception {
if (namespace == null || namespace.trim().isEmpty())
namespace = "ns" + counter.getAndIncrement();
NamespaceDescriptor nd = NamespaceDescriptor.create(namespace).build();
testUtil.getAdmin().createNamespace(nd);
return nd;
}

View File

@ -20,6 +20,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.testclassification.LargeTests;
@ -139,6 +140,63 @@ public class TestSpaceQuotaRemoval {
helper.verifyNoViolation(tn, put);
}
@Test
public void testDeleteTableUsageSnapshotsForNamespace() throws Exception {
Put put = new Put(Bytes.toBytes("to_reject"));
put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
Bytes.toBytes("reject"));
SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
//Create a namespace
String ns1 = "nsnew";
NamespaceDescriptor nsd = helper.createNamespace(ns1);
//Create 2nd namespace with name similar to ns1
String ns2 = ns1 + "test";
NamespaceDescriptor nsd2 = helper.createNamespace(ns2);
// Do puts until we violate space policy on table tn1 in namesapce ns1
final TableName tn1 = helper.writeUntilViolationAndVerifyViolationInNamespace(ns1, policy, put);
// Do puts until we violate space policy on table tn2 in namespace ns2
final TableName tn2 = helper.writeUntilViolationAndVerifyViolationInNamespace(ns2, policy, put);
// Now, remove the quota from namespace ns1 which will remove table usage snapshots for ns1
helper.removeQuotaFromNamespace(ns1);
// Verify that table usage snapshot for table tn2 in namespace ns2 exist
helper.verifyTableUsageSnapshotForSpaceQuotaExist(tn2);
// Put a new row on tn2: should violate as space quota exists on namespace ns2
helper.verifyViolation(policy, tn2, put);
// Put a new row on tn1: should not violate as quota settings removed from namespace ns1
helper.verifyNoViolation(tn1, put);
}
@Test
public void testSetNamespaceSizeQuotaAndThenRemove() throws Exception {
Put put = new Put(Bytes.toBytes("to_reject"));
put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
Bytes.toBytes("reject"));
SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
//Create namespace
NamespaceDescriptor nsd = helper.createNamespace();
String ns = nsd.getName();
// Do puts until we violate space policy on table tn1
final TableName tn1 = helper.writeUntilViolationAndVerifyViolationInNamespace(ns, policy, put);
// Now, remove the quota from namespace
helper.removeQuotaFromNamespace(ns);
// Put a new row now on tn1: should not violate as quota settings removed from namespace
helper.verifyNoViolation(tn1, put);
}
private void setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy policy)
throws Exception {
Put put = new Put(Bytes.toBytes("to_reject"));