HBASE-22146 Removing a namespace-level space quota does not remove policies against contained tables
Closes #1935 Signed-off-by: Josh Elser <elserj@apache.org>
This commit is contained in:
parent
a6e3db5ba5
commit
8191fbdd7d
|
@ -628,6 +628,34 @@ public class QuotaTableUtil {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove table usage snapshots (u:p columns) for the namespace passed
|
||||
* @param connection connection to re-use
|
||||
* @param namespace the namespace to fetch the list of table usage snapshots
|
||||
*/
|
||||
static void deleteTableUsageSnapshotsForNamespace(Connection connection, String namespace)
|
||||
throws IOException {
|
||||
Scan s = new Scan();
|
||||
//Get rows for all tables in namespace
|
||||
s.setRowPrefixFilter(Bytes.add(QUOTA_TABLE_ROW_KEY_PREFIX, Bytes.toBytes(namespace + TableName.NAMESPACE_DELIM)));
|
||||
//Scan for table usage column (u:p) in quota table
|
||||
s.addColumn(QUOTA_FAMILY_USAGE,QUOTA_QUALIFIER_POLICY);
|
||||
//Scan for table quota column (q:s) if table has a space quota defined
|
||||
s.addColumn(QUOTA_FAMILY_INFO,QUOTA_QUALIFIER_SETTINGS);
|
||||
try (Table quotaTable = connection.getTable(QUOTA_TABLE_NAME);
|
||||
ResultScanner rs = quotaTable.getScanner(s)) {
|
||||
for (Result r : rs) {
|
||||
byte[] data = r.getValue(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS);
|
||||
//if table does not have a table space quota defined, delete table usage column (u:p)
|
||||
if (data == null) {
|
||||
Delete delete = new Delete(r.getRow());
|
||||
delete.addColumns(QUOTA_FAMILY_USAGE,QUOTA_QUALIFIER_POLICY);
|
||||
quotaTable.delete(delete);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the computed size of all snapshots against tables in a namespace for space quotas.
|
||||
*/
|
||||
|
|
|
@ -266,6 +266,14 @@ public class QuotaUtil extends QuotaTableUtil {
|
|||
if (qualifier != null) {
|
||||
delete.addColumns(QUOTA_FAMILY_INFO, qualifier);
|
||||
}
|
||||
if (isNamespaceRowKey(rowKey)) {
|
||||
String ns = getNamespaceFromRowKey(rowKey);
|
||||
Quotas namespaceQuota = getNamespaceQuota(connection,ns);
|
||||
if (namespaceQuota != null && namespaceQuota.hasSpace()) {
|
||||
// When deleting namespace space quota, also delete table usage(u:p) snapshots
|
||||
deleteTableUsageSnapshotsForNamespace(connection, ns);
|
||||
}
|
||||
}
|
||||
doDelete(connection, delete);
|
||||
}
|
||||
|
||||
|
|
|
@ -158,6 +158,28 @@ public class SpaceQuotaHelperForTests {
|
|||
return tn;
|
||||
}
|
||||
|
||||
|
||||
TableName writeUntilViolationAndVerifyViolationInNamespace(
|
||||
String ns, SpaceViolationPolicy policyToViolate, Mutation m) throws Exception {
|
||||
final TableName tn = writeUntilViolationInNamespace(ns, policyToViolate);
|
||||
verifyViolation(policyToViolate, tn, m);
|
||||
return tn;
|
||||
}
|
||||
|
||||
TableName writeUntilViolationInNamespace(String ns, SpaceViolationPolicy policyToViolate) throws Exception {
|
||||
TableName tn = createTableWithRegions(ns,10);
|
||||
|
||||
setQuotaLimit(ns, policyToViolate, 4L);
|
||||
|
||||
// Write more data than should be allowed and flush it to disk
|
||||
writeData(tn, 5L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
|
||||
|
||||
// This should be sufficient time for the chores to run and see the change.
|
||||
Thread.sleep(5000);
|
||||
|
||||
return tn;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that the given policy on the given table has been violated
|
||||
*/
|
||||
|
@ -260,6 +282,19 @@ public class SpaceQuotaHelperForTests {
|
|||
assertTrue("Expected to succeed in writing data to a table not having quota ", sawSuccess);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies that table usage snapshot exists for the table
|
||||
*/
|
||||
void verifyTableUsageSnapshotForSpaceQuotaExist(TableName tn) throws Exception {
|
||||
boolean sawUsageSnapshot = false;
|
||||
try (Table quotaTable = testUtil.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
|
||||
Scan s = QuotaTableUtil.makeQuotaSnapshotScanForTable(tn);
|
||||
ResultScanner rs = quotaTable.getScanner(s);
|
||||
sawUsageSnapshot = (rs.next() != null);
|
||||
}
|
||||
assertTrue("Expected to succeed in getting table usage snapshots for space quota", sawUsageSnapshot);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the given quota (policy & limit) on the passed table.
|
||||
*/
|
||||
|
@ -271,6 +306,17 @@ public class SpaceQuotaHelperForTests {
|
|||
LOG.debug("Quota limit set for table = {}, limit = {}", tn, sizeLimit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the given quota (policy & limit) on the passed namespace.
|
||||
*/
|
||||
void setQuotaLimit(String ns, SpaceViolationPolicy policy, long sizeInMBs)
|
||||
throws Exception {
|
||||
final long sizeLimit = sizeInMBs * SpaceQuotaHelperForTests.ONE_MEGABYTE;
|
||||
QuotaSettings settings = QuotaSettingsFactory.limitNamespaceSpace(ns, sizeLimit, policy);
|
||||
testUtil.getAdmin().setQuota(settings);
|
||||
LOG.debug("Quota limit set for namespace = {}, limit = {}", ns, sizeLimit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes the space quota from the given table
|
||||
*/
|
||||
|
@ -280,6 +326,16 @@ public class SpaceQuotaHelperForTests {
|
|||
LOG.debug("Space quota settings removed from the table ", tn);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes the space quota from the given namespace
|
||||
*/
|
||||
void removeQuotaFromNamespace(String ns) throws Exception {
|
||||
QuotaSettings removeQuota = QuotaSettingsFactory.removeNamespaceSpaceLimit(ns);
|
||||
Admin admin = testUtil.getAdmin();
|
||||
admin.setQuota(removeQuota);
|
||||
LOG.debug("Space quota settings removed from the namespace ", ns);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes all quotas defined in the HBase quota table.
|
||||
*/
|
||||
|
@ -411,7 +467,13 @@ public class SpaceQuotaHelperForTests {
|
|||
}
|
||||
|
||||
NamespaceDescriptor createNamespace() throws Exception {
|
||||
NamespaceDescriptor nd = NamespaceDescriptor.create("ns" + counter.getAndIncrement()).build();
|
||||
return createNamespace(null);
|
||||
}
|
||||
|
||||
NamespaceDescriptor createNamespace(String namespace) throws Exception {
|
||||
if (namespace == null || namespace.trim().isEmpty())
|
||||
namespace = "ns" + counter.getAndIncrement();
|
||||
NamespaceDescriptor nd = NamespaceDescriptor.create(namespace).build();
|
||||
testUtil.getAdmin().createNamespace(nd);
|
||||
return nd;
|
||||
}
|
||||
|
|
|
@ -20,6 +20,7 @@ import java.util.concurrent.atomic.AtomicLong;
|
|||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.NamespaceDescriptor;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.client.Put;
|
||||
import org.apache.hadoop.hbase.testclassification.LargeTests;
|
||||
|
@ -139,6 +140,63 @@ public class TestSpaceQuotaRemoval {
|
|||
helper.verifyNoViolation(tn, put);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeleteTableUsageSnapshotsForNamespace() throws Exception {
|
||||
Put put = new Put(Bytes.toBytes("to_reject"));
|
||||
put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
|
||||
Bytes.toBytes("reject"));
|
||||
|
||||
SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
|
||||
|
||||
//Create a namespace
|
||||
String ns1 = "nsnew";
|
||||
NamespaceDescriptor nsd = helper.createNamespace(ns1);
|
||||
|
||||
//Create 2nd namespace with name similar to ns1
|
||||
String ns2 = ns1 + "test";
|
||||
NamespaceDescriptor nsd2 = helper.createNamespace(ns2);
|
||||
|
||||
// Do puts until we violate space policy on table tn1 in namesapce ns1
|
||||
final TableName tn1 = helper.writeUntilViolationAndVerifyViolationInNamespace(ns1, policy, put);
|
||||
|
||||
// Do puts until we violate space policy on table tn2 in namespace ns2
|
||||
final TableName tn2 = helper.writeUntilViolationAndVerifyViolationInNamespace(ns2, policy, put);
|
||||
|
||||
// Now, remove the quota from namespace ns1 which will remove table usage snapshots for ns1
|
||||
helper.removeQuotaFromNamespace(ns1);
|
||||
|
||||
// Verify that table usage snapshot for table tn2 in namespace ns2 exist
|
||||
helper.verifyTableUsageSnapshotForSpaceQuotaExist(tn2);
|
||||
|
||||
// Put a new row on tn2: should violate as space quota exists on namespace ns2
|
||||
helper.verifyViolation(policy, tn2, put);
|
||||
|
||||
// Put a new row on tn1: should not violate as quota settings removed from namespace ns1
|
||||
helper.verifyNoViolation(tn1, put);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSetNamespaceSizeQuotaAndThenRemove() throws Exception {
|
||||
Put put = new Put(Bytes.toBytes("to_reject"));
|
||||
put.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
|
||||
Bytes.toBytes("reject"));
|
||||
|
||||
SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
|
||||
|
||||
//Create namespace
|
||||
NamespaceDescriptor nsd = helper.createNamespace();
|
||||
String ns = nsd.getName();
|
||||
|
||||
// Do puts until we violate space policy on table tn1
|
||||
final TableName tn1 = helper.writeUntilViolationAndVerifyViolationInNamespace(ns, policy, put);
|
||||
|
||||
// Now, remove the quota from namespace
|
||||
helper.removeQuotaFromNamespace(ns);
|
||||
|
||||
// Put a new row now on tn1: should not violate as quota settings removed from namespace
|
||||
helper.verifyNoViolation(tn1, put);
|
||||
}
|
||||
|
||||
private void setQuotaAndThenRemoveInOneAmongTwoTables(SpaceViolationPolicy policy)
|
||||
throws Exception {
|
||||
Put put = new Put(Bytes.toBytes("to_reject"));
|
||||
|
|
Loading…
Reference in New Issue