Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-13829 Add more ThrottleType (Guanghao Zhang)

This commit is contained in:
tedyu 2015-06-09 06:42:55 -07:00
parent 487e4aa74f
commit 6cc42c8cd1
9 changed files with 295 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
View File

@ -2905,6 +2905,10 @@ public final class ProtobufUtil {
switch (proto) {
case REQUEST_NUMBER: return ThrottleType.REQUEST_NUMBER;
case REQUEST_SIZE: return ThrottleType.REQUEST_SIZE;
case WRITE_NUMBER: return ThrottleType.WRITE_NUMBER;
case WRITE_SIZE: return ThrottleType.WRITE_SIZE;
case READ_NUMBER: return ThrottleType.READ_NUMBER;
case READ_SIZE: return ThrottleType.READ_SIZE;
}
throw new RuntimeException("Invalid ThrottleType " + proto);
}
@ -2919,6 +2923,10 @@ public final class ProtobufUtil {
switch (type) {
case REQUEST_NUMBER: return QuotaProtos.ThrottleType.REQUEST_NUMBER;
case REQUEST_SIZE: return QuotaProtos.ThrottleType.REQUEST_SIZE;
case WRITE_NUMBER: return QuotaProtos.ThrottleType.WRITE_NUMBER;
case WRITE_SIZE: return QuotaProtos.ThrottleType.WRITE_SIZE;
case READ_NUMBER: return QuotaProtos.ThrottleType.READ_NUMBER;
case READ_SIZE: return QuotaProtos.ThrottleType.READ_SIZE;
}
throw new RuntimeException("Invalid ThrottleType " + type);
}

16
hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
View File

@ -105,6 +105,22 @@ public class QuotaSettingsFactory {
settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace,
ThrottleType.REQUEST_SIZE, throttle.getReqSize()));
}
if (throttle.hasWriteNum()) {
settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace,
ThrottleType.WRITE_NUMBER, throttle.getWriteNum()));
}
if (throttle.hasWriteSize()) {
settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace,
ThrottleType.WRITE_SIZE, throttle.getWriteSize()));
}
if (throttle.hasReadNum()) {
settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace,
ThrottleType.READ_NUMBER, throttle.getReadNum()));
}
if (throttle.hasReadSize()) {
settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace,
ThrottleType.READ_SIZE, throttle.getReadSize()));
}
return settings;
}

4
hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottleSettings.java
View File

@ -74,9 +74,13 @@ class ThrottleSettings extends QuotaSettings {
if (timedQuota.hasSoftLimit()) {
switch (getThrottleType()) {
case REQUEST_NUMBER:
case WRITE_NUMBER:
case READ_NUMBER:
builder.append(String.format("%dreq", timedQuota.getSoftLimit()));
break;
case REQUEST_SIZE:
case WRITE_SIZE:
case READ_SIZE:
builder.append(sizeToString(timedQuota.getSoftLimit()));
break;
}

14
hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottleType.java
View File

@ -26,9 +26,21 @@ import org.apache.hadoop.hbase.classification.InterfaceStability;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public enum ThrottleType {
/** Throttling based on the number of request per time-unit */
/** Throttling based on the number of requests per time-unit */
REQUEST_NUMBER,
/** Throttling based on the read+write data size */
REQUEST_SIZE,
/** Throttling based on the number of write requests per time-unit */
WRITE_NUMBER,
/** Throttling based on the write data size */
WRITE_SIZE,
/** Throttling based on the number of read requests per time-unit */
READ_NUMBER,
/** Throttling based on the read data size */
READ_SIZE,
}

53
hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java
View File

@ -75,6 +75,54 @@ public class TestQuotaAdmin {
TEST_UTIL.shutdownMiniCluster();
}
@Test
public void testThrottleType() throws Exception {
Admin admin = TEST_UTIL.getHBaseAdmin();
String userName = User.getCurrent().getShortName();
admin.setQuota(QuotaSettingsFactory
.throttleUser(userName, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
admin.setQuota(QuotaSettingsFactory
.throttleUser(userName, ThrottleType.WRITE_NUMBER, 12, TimeUnit.MINUTES));
admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, true));
try (QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration())) {
int countThrottle = 0;
int countGlobalBypass = 0;
for (QuotaSettings settings: scanner) {
switch (settings.getQuotaType()) {
case THROTTLE:
ThrottleSettings throttle = (ThrottleSettings)settings;
if (throttle.getSoftLimit() == 6) {
assertEquals(ThrottleType.READ_NUMBER, throttle.getThrottleType());
} else if (throttle.getSoftLimit() == 12) {
assertEquals(ThrottleType.WRITE_NUMBER, throttle.getThrottleType());
} else {
fail("should not come here, because don't set quota with this limit");
}
assertEquals(userName, throttle.getUserName());
assertEquals(null, throttle.getTableName());
assertEquals(null, throttle.getNamespace());
assertEquals(TimeUnit.MINUTES, throttle.getTimeUnit());
countThrottle++;
break;
case GLOBAL_BYPASS:
countGlobalBypass++;
break;
default:
fail("unexpected settings type: " + settings.getQuotaType());
}
}
assertEquals(2, countThrottle);
assertEquals(1, countGlobalBypass);
}
admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
assertNumResults(1, null);
admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, false));
assertNumResults(0, null);
}
@Test
public void testSimpleScan() throws Exception {
Admin admin = TEST_UTIL.getHBaseAdmin();
@ -84,8 +132,7 @@ public class TestQuotaAdmin {
.throttleUser(userName, ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES));
admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, true));
QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration());
try {
try (QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration())) {
int countThrottle = 0;
int countGlobalBypass = 0;
for (QuotaSettings settings: scanner) {
@ -109,8 +156,6 @@ public class TestQuotaAdmin {
}
assertEquals(1, countThrottle);
assertEquals(1, countGlobalBypass);
} finally {
scanner.close();
}
admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));

176
hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java
View File

@ -139,6 +139,38 @@ public class TestQuotaThrottle {
assertEquals(60, doGets(60, tables));
}
@Test(timeout=60000)
public void testUserGlobalReadAndWriteThrottle() throws Exception {
final Admin admin = TEST_UTIL.getHBaseAdmin();
final String userName = User.getCurrent().getShortName();
// Add 6req/min limit for read request
admin.setQuota(QuotaSettingsFactory
.throttleUser(userName, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
triggerUserCacheRefresh(false, TABLE_NAMES);
// not limit for write request and should execute at max 6 read requests
assertEquals(60, doPuts(60, tables));
assertEquals(6, doGets(100, tables));
waitMinuteQuota();
// Add 6req/min limit for write request
admin.setQuota(QuotaSettingsFactory
.throttleUser(userName, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
triggerUserCacheRefresh(false, TABLE_NAMES);
// should execute at max 6 read requests and at max 6 write write requests
assertEquals(6, doGets(100, tables));
assertEquals(6, doPuts(60, tables));
// Remove all the limits
admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName));
triggerUserCacheRefresh(true, TABLE_NAMES);
assertEquals(60, doPuts(60, tables));
assertEquals(60, doGets(60, tables));
}
@Test(timeout=60000)
public void testUserTableThrottle() throws Exception {
final Admin admin = TEST_UTIL.getHBaseAdmin();
@ -164,6 +196,47 @@ public class TestQuotaThrottle {
assertEquals(60, doGets(60, tables));
}
@Test(timeout=60000)
public void testUserTableReadAndWriteThrottle() throws Exception {
final Admin admin = TEST_UTIL.getHBaseAdmin();
final String userName = User.getCurrent().getShortName();
// Add 6req/min limit for write request on tables[0]
admin.setQuota(QuotaSettingsFactory
.throttleUser(userName, TABLE_NAMES[0], ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
triggerUserCacheRefresh(false, TABLE_NAMES[0]);
// should execute at max 6 write requests and have no limit for read request
assertEquals(6, doPuts(100, tables[0]));
assertEquals(60, doGets(60, tables[0]));
// no limit on tables[1]
assertEquals(60, doPuts(60, tables[1]));
assertEquals(60, doGets(60, tables[1]));
// wait a minute and you should get other 6 write requests executed
waitMinuteQuota();
// Add 6req/min limit for read request on tables[0]
admin.setQuota(QuotaSettingsFactory
.throttleUser(userName, TABLE_NAMES[0], ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
triggerUserCacheRefresh(false, TABLE_NAMES[0]);
// should execute at max 6 read requests and at max 6 write requests
assertEquals(6, doPuts(100, tables[0]));
assertEquals(6, doGets(60, tables[0]));
// no limit on tables[1]
assertEquals(30, doPuts(30, tables[1]));
assertEquals(30, doGets(30, tables[1]));
// Remove all the limits
admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, TABLE_NAMES[0]));
triggerUserCacheRefresh(true, TABLE_NAMES);
assertEquals(60, doPuts(60, tables));
assertEquals(60, doGets(60, tables));
}
@Test(timeout=60000)
public void testUserNamespaceThrottle() throws Exception {
final Admin admin = TEST_UTIL.getHBaseAdmin();
@ -189,6 +262,39 @@ public class TestQuotaThrottle {
assertEquals(60, doGets(60, tables));
}
@Test(timeout=60000)
public void testUserNamespaceReadAndWriteThrottle() throws Exception {
final Admin admin = TEST_UTIL.getHBaseAdmin();
final String userName = User.getCurrent().getShortName();
final String NAMESPACE = "default";
// Add 6req/min limit for read request
admin.setQuota(QuotaSettingsFactory
.throttleUser(userName, NAMESPACE, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
triggerUserCacheRefresh(false, TABLE_NAMES[0]);
// should execute at max 6 read requests and have no limit for write request
assertEquals(6, doGets(60, tables[0]));
assertEquals(60, doPuts(60, tables[0]));
waitMinuteQuota();
// Add 6req/min limit for write request, too
admin.setQuota(QuotaSettingsFactory
.throttleUser(userName, NAMESPACE, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
triggerUserCacheRefresh(false, TABLE_NAMES[0]);
// should execute at max 6 read requests and at max 6 write requests
assertEquals(6, doGets(60, tables[0]));
assertEquals(6, doPuts(60, tables[0]));
// Remove all the limits
admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName, NAMESPACE));
triggerUserCacheRefresh(true, TABLE_NAMES);
assertEquals(60, doPuts(60, tables));
assertEquals(60, doGets(60, tables));
}
@Test(timeout=60000)
public void testTableGlobalThrottle() throws Exception {
final Admin admin = TEST_UTIL.getHBaseAdmin();
@ -213,6 +319,43 @@ public class TestQuotaThrottle {
assertEquals(80, doGets(80, tables[0], tables[1]));
}
@Test(timeout=60000)
public void testTableGlobalReadAndWriteThrottle() throws Exception {
final Admin admin = TEST_UTIL.getHBaseAdmin();
// Add 6req/min limit for read request
admin.setQuota(QuotaSettingsFactory
.throttleTable(TABLE_NAMES[0], ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
triggerTableCacheRefresh(false, TABLE_NAMES[0]);
// should execute at max 6 read requests and have no limit for write request
assertEquals(6, doGets(100, tables[0]));
assertEquals(100, doPuts(100, tables[0]));
// should have no limits on tables[1]
assertEquals(30, doPuts(30, tables[1]));
assertEquals(30, doGets(30, tables[1]));
// wait a minute and you should get other 6 requests executed
waitMinuteQuota();
// Add 6req/min limit for write request, too
admin.setQuota(QuotaSettingsFactory
.throttleTable(TABLE_NAMES[0], ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
triggerTableCacheRefresh(false, TABLE_NAMES[0]);
// should execute at max 6 read requests and at max 6 write requests
assertEquals(6, doGets(100, tables[0]));
assertEquals(6, doPuts(100, tables[0]));
// should have no limits on tables[1]
assertEquals(30, doPuts(30, tables[1]));
assertEquals(30, doGets(30, tables[1]));
// Remove all the limits
admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0]));
triggerTableCacheRefresh(true, TABLE_NAMES[0]);
assertEquals(80, doGets(80, tables[0], tables[1]));
}
@Test(timeout=60000)
public void testNamespaceGlobalThrottle() throws Exception {
final Admin admin = TEST_UTIL.getHBaseAdmin();
@ -235,6 +378,37 @@ public class TestQuotaThrottle {
assertEquals(40, doPuts(40, tables[0]));
}
@Test(timeout=60000)
public void testNamespaceGlobalReadAndWriteThrottle() throws Exception {
final Admin admin = TEST_UTIL.getHBaseAdmin();
final String NAMESPACE = "default";
// Add 6req/min limit for write request
admin.setQuota(QuotaSettingsFactory
.throttleNamespace(NAMESPACE, ThrottleType.WRITE_NUMBER, 6, TimeUnit.MINUTES));
triggerNamespaceCacheRefresh(false, TABLE_NAMES[0]);
// should execute at max 6 write requests and no limit for read request
assertEquals(6, doPuts(100, tables[0]));
assertEquals(100, doGets(100, tables[0]));
// wait a minute and you should get other 6 requests executed
waitMinuteQuota();
// Add 6req/min limit for read request, too
admin.setQuota(QuotaSettingsFactory
.throttleNamespace(NAMESPACE, ThrottleType.READ_NUMBER, 6, TimeUnit.MINUTES));
triggerNamespaceCacheRefresh(false, TABLE_NAMES[0]);
// should execute at max 6 write requests and at max 6 read requests
assertEquals(6, doPuts(100, tables[0]));
assertEquals(6, doGets(100, tables[0]));
admin.setQuota(QuotaSettingsFactory.unthrottleNamespace(NAMESPACE));
triggerNamespaceCacheRefresh(true, TABLE_NAMES[0]);
assertEquals(40, doPuts(40, tables[0]));
}
@Test(timeout=60000)
public void testUserAndTableThrottle() throws Exception {
final Admin admin = TEST_UTIL.getHBaseAdmin();
@ -389,6 +563,8 @@ public class TestQuotaThrottle {
QuotaCache quotaCache = quotaManager.getQuotaCache();
quotaCache.triggerCacheRefresh();
// sleep for cache update
Thread.sleep(250);
for (TableName table: tables) {
quotaCache.getTableLimiter(table);

5
hbase-shell/src/main/ruby/hbase/quotas.rb
View File

@ -30,6 +30,8 @@ module HBaseQuotasConstants
THROTTLE_TYPE = 'THROTTLE_TYPE'
THROTTLE = 'THROTTLE'
REQUEST = 'REQUEST'
WRITE = 'WRITE'
READ = 'READ'
end
module Hbase
@ -46,6 +48,7 @@ module Hbase
def throttle(args)
raise(ArgumentError, "Arguments should be a Hash") unless args.kind_of?(Hash)
type = args.fetch(THROTTLE_TYPE, REQUEST)
args.delete(THROTTLE_TYPE)
type, limit, time_unit = _parse_limit(args.delete(LIMIT), ThrottleType, type)
if args.has_key?(USER)
user = args.delete(USER)
@ -213,4 +216,4 @@ module Hbase
return value
end
end
end
end

16
hbase-shell/src/main/ruby/shell/commands/set_quota.rb
View File

@ -26,21 +26,31 @@ Set a quota for a user, table, or namespace.
Syntax : set_quota TYPE => <type>, <args>
TYPE => THROTTLE
The request limit can be expressed using the form 100req/sec, 100req/min
and the size limit can be expressed using the form 100k/sec, 100M/min
with (B, K, M, G, T, P) as valid size unit and (sec, min, hour, day) as valid time unit.
User can either set quota on read, write or on both the requests together(i.e., read+write)
The read, write, or read+write(default throttle type) request limit can be expressed using
the form 100req/sec, 100req/min and the read, write, read+write(default throttle type) limit
can be expressed using the form 100k/sec, 100M/min with (B, K, M, G, T, P) as valid size unit
and (sec, min, hour, day) as valid time unit.
Currently the throttle limit is per machine - a limit of 100req/min
means that each machine can execute 100req/min.
For example:
hbase> set_quota TYPE => THROTTLE, USER => 'u1', LIMIT => '10req/sec'
hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => READ, USER => 'u1', LIMIT => '10req/sec'
hbase> set_quota TYPE => THROTTLE, USER => 'u1', LIMIT => '10M/sec'
hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => WRITE, USER => 'u1', LIMIT => '10M/sec'
hbase> set_quota TYPE => THROTTLE, USER => 'u1', TABLE => 't2', LIMIT => '5K/min'
hbase> set_quota TYPE => THROTTLE, USER => 'u1', NAMESPACE => 'ns2', LIMIT => NONE
hbase> set_quota TYPE => THROTTLE, NAMESPACE => 'ns1', LIMIT => '10req/sec'
hbase> set_quota TYPE => THROTTLE, TABLE => 't1', LIMIT => '10M/sec'
hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => WRITE, TABLE => 't1', LIMIT => '10M/sec'
hbase> set_quota TYPE => THROTTLE, USER => 'u1', LIMIT => NONE
hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => WRITE, USER => 'u1', LIMIT => NONE
hbase> set_quota USER => 'u1', GLOBAL_BYPASS => true
EOF
end

14
src/main/asciidoc/_chapters/ops_mgt.adoc
View File

@ -1622,7 +1622,7 @@ handling multiple workloads:
HBASE-11598 introduces quotas, which allow you to throttle requests based on
the following limits:
. <<request-quotas,The number or size of requests in a given timeframe>>
. <<request-quotas,The number or size of requests(read, write, or read+write) in a given timeframe>>
. <<namespace-quotas,The number of tables allowed in a namespace>>
These limits can be enforced for a specified user, table, or namespace.
@ -1633,6 +1633,7 @@ Quotas are disabled by default. To enable the feature, set the `hbase.quota.enab
property to `true` in _hbase-site.xml_ file for all cluster nodes.
.General Quota Syntax
. THROTTLE_TYPE can be expressed as READ, WRITE, or the default type(read + write).
. Timeframes can be expressed in the following units: `sec`, `min`, `hour`, `day`
. Request sizes can be expressed in the following units: `B` (bytes), `K` (kilobytes),
`M` (megabytes), `G` (gigabytes), `T` (terabytes), `P` (petabytes)
@ -1652,17 +1653,26 @@ in `hbase-site.xml`. This property is expressed in milliseconds and defaults to
# Limit user u1 to 10 requests per second
hbase> set_quota TYPE => THROTTLE, USER => 'u1', LIMIT => '10req/sec'
# Limit user u1 to 10 read requests per second
hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => READ, USER => 'u1', LIMIT => '10req/sec'
# Limit user u1 to 10 M per day everywhere
hbase> set_quota TYPE => THROTTLE, USER => 'u1', LIMIT => '10M/day'
# Limit user u1 to 10 M write size per sec
hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => WRITE, USER => 'u1', LIMIT => '10M/sec'
# Limit user u1 to 5k per minute on table t2
hbase> set_quota TYPE => THROTTLE, USER => 'u1', TABLE => 't2', LIMIT => '5K/min'
# Limit user u1 to 10 read requests per sec on table t2
hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => READ, USER => 'u1', TABLE => 't2', LIMIT => '10req/sec'
# Remove an existing limit from user u1 on namespace ns2
hbase> set_quota TYPE => THROTTLE, USER => 'u1', NAMESPACE => 'ns2', LIMIT => NONE
# Limit all users to 10 requests per hour on namespace ns1
hbase> set_quota TYPE => THROTTLE, NAMESPACE => 'ns1', LIMIT => '10req/shour'
hbase> set_quota TYPE => THROTTLE, NAMESPACE => 'ns1', LIMIT => '10req/hour'
# Limit all users to 10 T per hour on table t1
hbase> set_quota TYPE => THROTTLE, TABLE => 't1', LIMIT => '10T/hour'