From 9a55cbb2c1dfe5a13a6ceb323ac7edd23532f4b5 Mon Sep 17 00:00:00 2001 From: meiyi Date: Thu, 21 Feb 2019 09:28:49 +0800 Subject: [PATCH] HBASE-21783 Support exceed user/table/ns throttle quota if region server has available quota Signed-off-by: Guanghao Zhang --- .../org/apache/hadoop/hbase/client/Admin.java | 8 ++ .../hadoop/hbase/client/AsyncAdmin.java | 8 ++ .../hadoop/hbase/client/AsyncHBaseAdmin.java | 5 + .../client/ConnectionImplementation.java | 8 ++ .../hadoop/hbase/client/HBaseAdmin.java | 14 +++ .../hbase/client/RawAsyncHBaseAdmin.java | 16 +++ .../client/ShortCircuitMasterConnection.java | 8 ++ .../hadoop/hbase/quotas/QuotaTableUtil.java | 15 +++ .../src/main/protobuf/Master.proto | 12 ++ .../hbase/coprocessor/MasterObserver.java | 20 ++++ .../hbase/master/MasterCoprocessorHost.java | 19 +++ .../hbase/master/MasterRpcServices.java | 13 ++ .../hbase/quotas/DefaultOperationQuota.java | 54 ++++++--- .../hbase/quotas/ExceedOperationQuota.java | 102 ++++++++++++++++ .../hbase/quotas/MasterQuotaManager.java | 33 ++++- .../hadoop/hbase/quotas/QuotaCache.java | 15 +++ .../apache/hadoop/hbase/quotas/QuotaUtil.java | 86 +++++++++++++ .../quotas/RegionServerRpcQuotaManager.java | 12 +- .../security/access/AccessController.java | 6 + .../hbase/client/TestAsyncQuotaAdminApi.java | 6 + .../hadoop/hbase/quotas/TestQuotaAdmin.java | 42 +++++++ .../hbase/quotas/TestQuotaThrottle.java | 113 +++++++++++++++--- .../security/access/TestAccessController.java | 14 +++ hbase-shell/src/main/ruby/hbase/quotas.rb | 4 + hbase-shell/src/main/ruby/shell.rb | 2 + .../commands/disable_exceed_throttle_quota.rb | 40 +++++++ .../commands/enable_exceed_throttle_quota.rb | 50 ++++++++ .../src/test/ruby/hbase/quotas_test.rb | 10 ++ .../hbase/thrift2/client/ThriftAdmin.java | 6 + 29 files changed, 703 insertions(+), 38 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ExceedOperationQuota.java create mode 100644 hbase-shell/src/main/ruby/shell/commands/disable_exceed_throttle_quota.rb create mode 100644 hbase-shell/src/main/ruby/shell/commands/enable_exceed_throttle_quota.rb diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index deaea8c29c5..054702a1b30 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -2812,6 +2812,14 @@ public interface Admin extends Abortable, Closeable { */ boolean isRpcThrottleEnabled() throws IOException; + /** + * Switch the exceed throttle quota. If enabled, user/table/namespace throttle quota + * can be exceeded if region server has availble quota. + * @param enable Set to true to enable, false to disable. + * @return Previous exceed throttle enabled value + */ + boolean exceedThrottleQuotaSwitch(final boolean enable) throws IOException; + /** * Fetches the table sizes on the filesystem as tracked by the HBase Master. */ diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index 9abfe23763b..b45a040393d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -1303,6 +1303,14 @@ public interface AsyncAdmin { */ CompletableFuture isRpcThrottleEnabled(); + /** + * Switch the exceed throttle quota. If enabled, user/table/namespace throttle quota + * can be exceeded if region server has availble quota. + * @param enable Set to true to enable, false to disable. + * @return Previous exceed throttle enabled value + */ + CompletableFuture exceedThrottleQuotaSwitch(boolean enable); + /** * Fetches the table sizes on the filesystem as tracked by the HBase Master. */ diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index f39fe36768e..960b72a37e2 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -770,6 +770,11 @@ class AsyncHBaseAdmin implements AsyncAdmin { return wrap(rawAdmin.isRpcThrottleEnabled()); } + @Override + public CompletableFuture exceedThrottleQuotaSwitch(boolean enable) { + return wrap(rawAdmin.exceedThrottleQuotaSwitch(enable)); + } + @Override public CompletableFuture> getSpaceQuotaTableSizes() { return wrap(rawAdmin.getSpaceQuotaTableSizes()); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java index 5df3c07c033..ff2ff2f650b 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java @@ -115,6 +115,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCa import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchExceedThrottleQuotaRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchExceedThrottleQuotaResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchRpcThrottleRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchRpcThrottleResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest; @@ -1776,6 +1778,12 @@ class ConnectionImplementation implements ClusterConnection, Closeable { return stub.isRpcThrottleEnabled(controller, request); } + @Override + public SwitchExceedThrottleQuotaResponse switchExceedThrottleQuota(RpcController controller, + SwitchExceedThrottleQuotaRequest request) throws ServiceException { + return stub.switchExceedThrottleQuota(controller, request); + } + @Override public AccessControlProtos.GrantResponse grant(RpcController controller, AccessControlProtos.GrantRequest request) throws ServiceException { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 769ddd79627..f7402184d96 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -4391,6 +4391,20 @@ public class HBaseAdmin implements Admin { }); } + @Override + public boolean exceedThrottleQuotaSwitch(final boolean enable) throws IOException { + return executeCallable(new MasterCallable(getConnection(), getRpcControllerFactory()) { + @Override + protected Boolean rpcCall() throws Exception { + return this.master + .switchExceedThrottleQuota(getRpcController(), + MasterProtos.SwitchExceedThrottleQuotaRequest.newBuilder() + .setExceedThrottleQuotaEnabled(enable).build()) + .getPreviousExceedThrottleQuotaEnabled(); + } + }); + } + @Override public Map getSpaceQuotaTableSizes() throws IOException { return executeCallable( diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index 0e3b9457e7f..8dc3b01941d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -257,6 +257,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SplitTable import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SplitTableRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.StopMasterRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.StopMasterResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchExceedThrottleQuotaRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchExceedThrottleQuotaResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchRpcThrottleRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchRpcThrottleResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableRequest; @@ -3683,6 +3685,20 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { return future; } + @Override + public CompletableFuture exceedThrottleQuotaSwitch(boolean enable) { + CompletableFuture future = this. newMasterCaller() + .action((controller, stub) -> this + . call( + controller, stub, + SwitchExceedThrottleQuotaRequest.newBuilder().setExceedThrottleQuotaEnabled(enable) + .build(), + (s, c, req, done) -> s.switchExceedThrottleQuota(c, req, done), + resp -> resp.getPreviousExceedThrottleQuotaEnabled())) + .call(); + return future; + } + @Override public CompletableFuture> getSpaceQuotaTableSizes() { return this.> newMasterCaller().action((controller, stub) -> this diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java index 02935548156..090bcf9db7d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java @@ -153,6 +153,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SplitTable import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SplitTableRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.StopMasterRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.StopMasterResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchExceedThrottleQuotaRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchExceedThrottleQuotaResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchRpcThrottleRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchRpcThrottleResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableRequest; @@ -669,6 +671,12 @@ public class ShortCircuitMasterConnection implements MasterKeepAliveConnection { return stub.isRpcThrottleEnabled(controller, request); } + @Override + public SwitchExceedThrottleQuotaResponse switchExceedThrottleQuota(RpcController controller, + SwitchExceedThrottleQuotaRequest request) throws ServiceException { + return stub.switchExceedThrottleQuota(controller, request); + } + @Override public GrantResponse grant(RpcController controller, GrantRequest request) throws ServiceException { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java index 42ab58f14cb..b932242a641 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java @@ -100,6 +100,8 @@ public class QuotaTableUtil { protected static final byte[] QUOTA_TABLE_ROW_KEY_PREFIX = Bytes.toBytes("t."); protected static final byte[] QUOTA_NAMESPACE_ROW_KEY_PREFIX = Bytes.toBytes("n."); protected static final byte[] QUOTA_REGION_SERVER_ROW_KEY_PREFIX = Bytes.toBytes("r."); + private static final byte[] QUOTA_EXCEED_THROTTLE_QUOTA_ROW_KEY = + Bytes.toBytes("exceedThrottleQuota"); /* * TODO: Setting specified region server quota isn't supported currently and the row key "r.all" @@ -359,6 +361,11 @@ public class QuotaTableUtil { parseUserResult(result, visitor); } else if (isRegionServerRowKey(row)) { parseRegionServerResult(result, visitor); + } else if (isExceedThrottleQuotaRowKey(row)) { + // skip exceed throttle quota row key + if (LOG.isDebugEnabled()) { + LOG.debug("Skip exceedThrottleQuota row-key when parse quota result"); + } } else { LOG.warn("unexpected row-key: " + Bytes.toString(row)); } @@ -698,6 +705,10 @@ public class QuotaTableUtil { return getRowKeyRegEx(QUOTA_REGION_SERVER_ROW_KEY_PREFIX, regionServer); } + protected static byte[] getExceedThrottleQuotaRowKey() { + return QUOTA_EXCEED_THROTTLE_QUOTA_ROW_KEY; + } + private static String getRowKeyRegEx(final byte[] prefix, final String regex) { return '^' + Pattern.quote(Bytes.toString(prefix)) + regex + '$'; } @@ -724,6 +735,10 @@ public class QuotaTableUtil { return Bytes.startsWith(key, QUOTA_REGION_SERVER_ROW_KEY_PREFIX); } + private static boolean isExceedThrottleQuotaRowKey(final byte[] key) { + return Bytes.equals(key, QUOTA_EXCEED_THROTTLE_QUOTA_ROW_KEY); + } + protected static String getRegionServerFromRowKey(final byte[] key) { return Bytes.toString(key, QUOTA_REGION_SERVER_ROW_KEY_PREFIX.length); } diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto index ec0b5439cfa..4ed0ad5c0e2 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto @@ -652,6 +652,14 @@ message IsRpcThrottleEnabledResponse { required bool rpc_throttle_enabled = 1; } +message SwitchExceedThrottleQuotaRequest { + required bool exceed_throttle_quota_enabled = 1; +} + +message SwitchExceedThrottleQuotaResponse { + required bool previous_exceed_throttle_quota_enabled = 1; +} + service MasterService { /** Used by the client to get the number of regions that have received the updated schema */ rpc GetSchemaAlterStatus(GetSchemaAlterStatusRequest) @@ -1016,6 +1024,10 @@ service MasterService { rpc IsRpcThrottleEnabled (IsRpcThrottleEnabledRequest) returns (IsRpcThrottleEnabledResponse); + /** Turn the exceed throttle quota on or off */ + rpc SwitchExceedThrottleQuota (SwitchExceedThrottleQuotaRequest) + returns (SwitchExceedThrottleQuotaResponse); + rpc Grant(GrantRequest) returns (GrantResponse); rpc Revoke(RevokeRequest) returns (RevokeResponse); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java index 68e69b95035..d8b7d4d486d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java @@ -1575,6 +1575,26 @@ public interface MasterObserver { final boolean rpcThrottleEnabled) throws IOException { } + /** + * Called before switching exceed throttle quota state. + * @param ctx the coprocessor instance's environment + * @param enable the exceed throttle quota value + */ + default void preSwitchExceedThrottleQuota(final ObserverContext ctx, + final boolean enable) throws IOException { + } + + /** + * Called after switching exceed throttle quota state. + * @param ctx the coprocessor instance's environment + * @param oldValue the previously exceed throttle quota value + * @param newValue the newly exceed throttle quota value + */ + default void postSwitchExceedThrottleQuota( + final ObserverContext ctx, final boolean oldValue, + final boolean newValue) throws IOException { + } + /** * Called before granting user permissions. * @param ctx the coprocessor instance's environment diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java index d999dae14c1..8764143d1b7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java @@ -1836,6 +1836,25 @@ public class MasterCoprocessorHost }); } + public void preSwitchExceedThrottleQuota(boolean enable) throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { + @Override + public void call(MasterObserver observer) throws IOException { + observer.preSwitchExceedThrottleQuota(this, enable); + } + }); + } + + public void postSwitchExceedThrottleQuota(final boolean oldValue, final boolean newValue) + throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { + @Override + public void call(MasterObserver observer) throws IOException { + observer.postSwitchExceedThrottleQuota(this, oldValue, newValue); + } + }); + } + public void preGrant(UserPermission userPermission, boolean mergeExistingPermissions) throws IOException { execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 1ff3f0eaa77..e5fc0b83c94 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -266,6 +266,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SplitTable import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SplitTableRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.StopMasterRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.StopMasterResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchExceedThrottleQuotaRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchExceedThrottleQuotaResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchRpcThrottleRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchRpcThrottleResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableRequest; @@ -2510,6 +2512,17 @@ public class MasterRpcServices extends RSRpcServices } } + @Override + public SwitchExceedThrottleQuotaResponse switchExceedThrottleQuota(RpcController controller, + SwitchExceedThrottleQuotaRequest request) throws ServiceException { + try { + master.checkInitialized(); + return master.getMasterQuotaManager().switchExceedThrottleQuota(request); + } catch (Exception e) { + throw new ServiceException(e); + } + } + @Override public GrantResponse grant(RpcController controller, GrantRequest request) throws ServiceException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/DefaultOperationQuota.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/DefaultOperationQuota.java index f9b3ca5c29b..6f0e5cd30a4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/DefaultOperationQuota.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/DefaultOperationQuota.java @@ -34,17 +34,27 @@ import org.apache.hadoop.hbase.client.Result; public class DefaultOperationQuota implements OperationQuota { private static final Logger LOG = LoggerFactory.getLogger(DefaultOperationQuota.class); - private final List limiters; + protected final List limiters; private final long writeCapacityUnit; private final long readCapacityUnit; - private long writeAvailable = 0; - private long readAvailable = 0; - private long writeConsumed = 0; - private long readConsumed = 0; - private long writeCapacityUnitConsumed = 0; - private long readCapacityUnitConsumed = 0; + // the available read/write quota size in bytes + protected long writeAvailable = 0; + protected long readAvailable = 0; + // estimated quota + protected long writeConsumed = 0; + protected long readConsumed = 0; + protected long writeCapacityUnitConsumed = 0; + protected long readCapacityUnitConsumed = 0; + // real consumed quota private final long[] operationSize; + // difference between estimated quota and real consumed quota used in close method + // to adjust quota amount. Also used by ExceedOperationQuota which is a subclass + // of DefaultOperationQuota + protected long writeDiff = 0; + protected long readDiff = 0; + protected long writeCapacityUnitDiff = 0; + protected long readCapacityUnitDiff = 0; public DefaultOperationQuota(final Configuration conf, final QuotaLimiter... limiters) { this(conf, Arrays.asList(limiters)); @@ -69,12 +79,7 @@ public class DefaultOperationQuota implements OperationQuota { @Override public void checkQuota(int numWrites, int numReads, int numScans) throws RpcThrottlingException { - writeConsumed = estimateConsume(OperationType.MUTATE, numWrites, 100); - readConsumed = estimateConsume(OperationType.GET, numReads, 100); - readConsumed += estimateConsume(OperationType.SCAN, numScans, 1000); - - writeCapacityUnitConsumed = calculateWriteCapacityUnit(writeConsumed); - readCapacityUnitConsumed = calculateReadCapacityUnit(readConsumed); + updateEstimateConsumeQuota(numWrites, numReads, numScans); writeAvailable = Long.MAX_VALUE; readAvailable = Long.MAX_VALUE; @@ -96,12 +101,12 @@ public class DefaultOperationQuota implements OperationQuota { @Override public void close() { // Adjust the quota consumed for the specified operation - long writeDiff = operationSize[OperationType.MUTATE.ordinal()] - writeConsumed; - long readDiff = operationSize[OperationType.GET.ordinal()] + writeDiff = operationSize[OperationType.MUTATE.ordinal()] - writeConsumed; + readDiff = operationSize[OperationType.GET.ordinal()] + operationSize[OperationType.SCAN.ordinal()] - readConsumed; - long writeCapacityUnitDiff = calculateWriteCapacityUnitDiff( + writeCapacityUnitDiff = calculateWriteCapacityUnitDiff( operationSize[OperationType.MUTATE.ordinal()], writeConsumed); - long readCapacityUnitDiff = calculateReadCapacityUnitDiff( + readCapacityUnitDiff = calculateReadCapacityUnitDiff( operationSize[OperationType.GET.ordinal()] + operationSize[OperationType.SCAN.ordinal()], readConsumed); @@ -140,6 +145,21 @@ public class DefaultOperationQuota implements OperationQuota { operationSize[OperationType.MUTATE.ordinal()] += QuotaUtil.calculateMutationSize(mutation); } + /** + * Update estimate quota(read/write size/capacityUnits) which will be consumed + * @param numWrites the number of write requests + * @param numReads the number of read requests + * @param numScans the number of scan requests + */ + protected void updateEstimateConsumeQuota(int numWrites, int numReads, int numScans) { + writeConsumed = estimateConsume(OperationType.MUTATE, numWrites, 100); + readConsumed = estimateConsume(OperationType.GET, numReads, 100); + readConsumed += estimateConsume(OperationType.SCAN, numScans, 1000); + + writeCapacityUnitConsumed = calculateWriteCapacityUnit(writeConsumed); + readCapacityUnitConsumed = calculateReadCapacityUnit(readConsumed); + } + private long estimateConsume(final OperationType type, int numReqs, long avgSize) { if (numReqs > 0) { return avgSize * numReqs; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ExceedOperationQuota.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ExceedOperationQuota.java new file mode 100644 index 00000000000..5c9541e3e2c --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ExceedOperationQuota.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.quotas; + +import org.apache.hadoop.conf.Configuration; +import org.apache.yetus.audience.InterfaceAudience; +import org.apache.yetus.audience.InterfaceStability; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/* + * Internal class used to check and consume quota if exceed throttle quota is enabled. Exceed + * throttle quota means, user can over consume user/namespace/table quota if region server has + * additional available quota because other users don't consume at the same time. + * + * There are some limits when enable exceed throttle quota: + * 1. Must set at least one read and one write region server throttle quota; + * 2. All region server throttle quotas must be in seconds time unit. Because once previous requests + * exceed their quota and consume region server quota, quota in other time units may be refilled in + * a long time, this may affect later requests. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class ExceedOperationQuota extends DefaultOperationQuota { + private static final Logger LOG = LoggerFactory.getLogger(ExceedOperationQuota.class); + private QuotaLimiter regionServerLimiter; + + public ExceedOperationQuota(final Configuration conf, QuotaLimiter regionServerLimiter, + final QuotaLimiter... limiters) { + super(conf, limiters); + this.regionServerLimiter = regionServerLimiter; + } + + @Override + public void checkQuota(int numWrites, int numReads, int numScans) throws RpcThrottlingException { + if (regionServerLimiter.isBypass()) { + // If region server limiter is bypass, which means no region server quota is set, check and + // throttle by all other quotas. In this condition, exceed throttle quota will not work. + LOG.warn("Exceed throttle quota is enabled but no region server quotas found"); + super.checkQuota(numWrites, numReads, numScans); + } else { + // 1. Update estimate quota which will be consumed + updateEstimateConsumeQuota(numWrites, numReads, numScans); + // 2. Check if region server limiter is enough. If not, throw RpcThrottlingException. + regionServerLimiter.checkQuota(numWrites, writeConsumed, numReads + numScans, readConsumed, + writeCapacityUnitConsumed, readCapacityUnitConsumed); + // 3. Check if other limiters are enough. If not, exceed other limiters because region server + // limiter is enough. + boolean exceed = false; + try { + super.checkQuota(numWrites, numReads, numScans); + } catch (RpcThrottlingException e) { + exceed = true; + if (LOG.isDebugEnabled()) { + LOG.debug("Read/Write requests num exceeds quota: writes:{} reads:{} scan:{}, " + + "try use region server quota", + numWrites, numReads, numScans); + } + } + // 4. Region server limiter is enough and grab estimated consume quota. + readAvailable = Math.max(readAvailable, regionServerLimiter.getReadAvailable()); + writeAvailable = Math.max(writeAvailable, regionServerLimiter.getWriteAvailable()); + regionServerLimiter.grabQuota(numWrites, writeConsumed, numReads + numScans, readConsumed, + writeCapacityUnitConsumed, writeCapacityUnitConsumed); + if (exceed) { + // 5. Other quota limiter is exceeded and has not been grabbed (because throw + // RpcThrottlingException in Step 3), so grab it. + for (final QuotaLimiter limiter : limiters) { + limiter.grabQuota(numWrites, writeConsumed, numReads + numScans, readConsumed, + writeCapacityUnitConsumed, writeCapacityUnitConsumed); + } + } + } + } + + @Override + public void close() { + super.close(); + if (writeDiff != 0) { + regionServerLimiter.consumeWrite(writeDiff, writeCapacityUnitDiff); + } + if (readDiff != 0) { + regionServerLimiter.consumeRead(readDiff, readCapacityUnitDiff); + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java index 389e1791dfb..ffef30eccbc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java @@ -58,6 +58,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsRpcThrot import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsRpcThrottleEnabledResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchExceedThrottleQuotaRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchExceedThrottleQuotaResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchRpcThrottleRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SwitchRpcThrottleResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.FileArchiveNotificationRequest; @@ -301,7 +303,7 @@ public class MasterQuotaManager implements RegionStateListener { @Override public void update(GlobalQuotaSettingsImpl quotaPojo) throws IOException { QuotaUtil.addNamespaceQuota(masterServices.getConnection(), namespace, - ((GlobalQuotaSettingsImpl) quotaPojo).toQuotas()); + quotaPojo.toQuotas()); } @Override public void delete() throws IOException { @@ -330,7 +332,7 @@ public class MasterQuotaManager implements RegionStateListener { @Override public void update(GlobalQuotaSettingsImpl quotaPojo) throws IOException { QuotaUtil.addRegionServerQuota(masterServices.getConnection(), regionServer, - ((GlobalQuotaSettingsImpl) quotaPojo).toQuotas()); + quotaPojo.toQuotas()); } @Override @@ -405,6 +407,33 @@ public class MasterQuotaManager implements RegionStateListener { } } + public SwitchExceedThrottleQuotaResponse + switchExceedThrottleQuota(SwitchExceedThrottleQuotaRequest request) throws IOException { + boolean enabled = request.getExceedThrottleQuotaEnabled(); + if (initialized) { + masterServices.getMasterCoprocessorHost().preSwitchExceedThrottleQuota(enabled); + boolean previousEnabled = + QuotaUtil.isExceedThrottleQuotaEnabled(masterServices.getConnection()); + if (previousEnabled == enabled) { + LOG.warn("Skip switch exceed throttle quota to {} because it's the same with old value", + enabled); + } else { + QuotaUtil.switchExceedThrottleQuota(masterServices.getConnection(), enabled); + LOG.info("{} switch exceed throttle quota from {} to {}", + masterServices.getClientIdAuditPrefix(), previousEnabled, enabled); + } + SwitchExceedThrottleQuotaResponse response = SwitchExceedThrottleQuotaResponse.newBuilder() + .setPreviousExceedThrottleQuotaEnabled(previousEnabled).build(); + masterServices.getMasterCoprocessorHost().postSwitchExceedThrottleQuota(previousEnabled, + enabled); + return response; + } else { + LOG.warn("Skip switch exceed throttle quota to {} because quota is disabled", enabled); + return SwitchExceedThrottleQuotaResponse.newBuilder() + .setPreviousExceedThrottleQuotaEnabled(false).build(); + } + } + private void setQuota(final SetQuotaRequest req, final SetQuotaOperations quotaOps) throws IOException, InterruptedException { if (req.hasRemoveAll() && req.getRemoveAll() == true) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java index 4736362b1dd..336d8c19027 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java @@ -71,6 +71,7 @@ public class QuotaCache implements Stoppable { private final ConcurrentHashMap userQuotaCache = new ConcurrentHashMap<>(); private final ConcurrentHashMap regionServerQuotaCache = new ConcurrentHashMap<>(); + private volatile boolean exceedThrottleQuotaEnabled = false; private final RegionServerServices rsServices; private QuotaRefresherChore refreshChore; @@ -158,6 +159,10 @@ public class QuotaCache implements Stoppable { return getQuotaState(this.regionServerQuotaCache, regionServer).getGlobalLimiter(); } + protected boolean isExceedThrottleQuotaEnabled() { + return exceedThrottleQuotaEnabled; + } + /** * Returns the QuotaState requested. If the quota info is not in cache an empty one will be * returned and the quota request will be enqueued for the next cache refresh. @@ -227,6 +232,7 @@ public class QuotaCache implements Stoppable { fetchTableQuotaState(); fetchUserQuotaState(); fetchRegionServerQuotaState(); + fetchExceedThrottleQuota(); lastUpdate = EnvironmentEdgeManager.currentTime(); } @@ -292,6 +298,15 @@ public class QuotaCache implements Stoppable { }); } + private void fetchExceedThrottleQuota() { + try { + QuotaCache.this.exceedThrottleQuotaEnabled = + QuotaUtil.isExceedThrottleQuotaEnabled(rsServices.getConnection()); + } catch (IOException e) { + LOG.warn("Unable to read if exceed throttle quota enabled from quota table", e); + } + } + private void fetch(final String type, final ConcurrentHashMap quotasMap, final Fetcher fetcher) { long now = EnvironmentEdgeManager.currentTime(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java index e562336db88..2bff4e9a333 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java @@ -19,12 +19,14 @@ package org.apache.hadoop.hbase.quotas; import java.io.IOException; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; @@ -39,12 +41,16 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.TimeUnit; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.TimedQuota; /** * Helper class to interact with the quota table @@ -153,6 +159,86 @@ public class QuotaUtil extends QuotaTableUtil { deleteQuotas(connection, getRegionServerRowKey(regionServer)); } + protected static void switchExceedThrottleQuota(final Connection connection, + boolean exceedThrottleQuotaEnabled) throws IOException { + if (exceedThrottleQuotaEnabled) { + checkRSQuotaToEnableExceedThrottle( + getRegionServerQuota(connection, QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY)); + } + + Put put = new Put(getExceedThrottleQuotaRowKey()); + put.addColumn(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS, + Bytes.toBytes(exceedThrottleQuotaEnabled)); + doPut(connection, put); + } + + private static void checkRSQuotaToEnableExceedThrottle(Quotas quotas) throws IOException { + if (quotas != null && quotas.hasThrottle()) { + Throttle throttle = quotas.getThrottle(); + // If enable exceed throttle quota, make sure that there are at least one read(req/read + + // num/size/cu) and one write(req/write + num/size/cu) region server throttle quotas. + boolean hasReadQuota = false; + boolean hasWriteQuota = false; + if (throttle.hasReqNum() || throttle.hasReqSize() || throttle.hasReqCapacityUnit()) { + hasReadQuota = true; + hasWriteQuota = true; + } + if (!hasReadQuota + && (throttle.hasReadNum() || throttle.hasReadSize() || throttle.hasReadCapacityUnit())) { + hasReadQuota = true; + } + if (!hasReadQuota) { + throw new DoNotRetryIOException( + "Please set at least one read region server quota before enable exceed throttle quota"); + } + if (!hasWriteQuota && (throttle.hasWriteNum() || throttle.hasWriteSize() + || throttle.hasWriteCapacityUnit())) { + hasWriteQuota = true; + } + if (!hasWriteQuota) { + throw new DoNotRetryIOException("Please set at least one write region server quota " + + "before enable exceed throttle quota"); + } + // If enable exceed throttle quota, make sure that region server throttle quotas are in + // seconds time unit. Because once previous requests exceed their quota and consume region + // server quota, quota in other time units may be refilled in a long time, this may affect + // later requests. + List> list = + Arrays.asList(Pair.newPair(throttle.hasReqNum(), throttle.getReqNum()), + Pair.newPair(throttle.hasReadNum(), throttle.getReadNum()), + Pair.newPair(throttle.hasWriteNum(), throttle.getWriteNum()), + Pair.newPair(throttle.hasReqSize(), throttle.getReqSize()), + Pair.newPair(throttle.hasReadSize(), throttle.getReadSize()), + Pair.newPair(throttle.hasWriteSize(), throttle.getWriteSize()), + Pair.newPair(throttle.hasReqCapacityUnit(), throttle.getReqCapacityUnit()), + Pair.newPair(throttle.hasReadCapacityUnit(), throttle.getReadCapacityUnit()), + Pair.newPair(throttle.hasWriteCapacityUnit(), throttle.getWriteCapacityUnit())); + for (Pair pair : list) { + if (pair.getFirst()) { + if (pair.getSecond().getTimeUnit() != TimeUnit.SECONDS) { + throw new DoNotRetryIOException("All region server quota must be " + + "in seconds time unit if enable exceed throttle quota"); + } + } + } + } else { + // If enable exceed throttle quota, make sure that region server quota is already set + throw new DoNotRetryIOException( + "Please set region server quota before enable exceed throttle quota"); + } + } + + protected static boolean isExceedThrottleQuotaEnabled(final Connection connection) + throws IOException { + Get get = new Get(getExceedThrottleQuotaRowKey()); + get.addColumn(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS); + Result result = doGet(connection, get); + if (result.isEmpty()) { + return false; + } + return Bytes.toBoolean(result.getValue(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS)); + } + private static void addQuotas(final Connection connection, final byte[] rowKey, final Quotas data) throws IOException { addQuotas(connection, rowKey, QUOTA_QUALIFIER_SETTINGS, data); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java index 99ff5162978..0f96de53565 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java @@ -138,14 +138,20 @@ public class RegionServerRpcQuotaManager { QuotaLimiter rsLimiter = quotaCache .getRegionServerQuotaLimiter(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY); useNoop &= tableLimiter.isBypass() && nsLimiter.isBypass() && rsLimiter.isBypass(); + boolean exceedThrottleQuotaEnabled = quotaCache.isExceedThrottleQuotaEnabled(); if (LOG.isTraceEnabled()) { LOG.trace("get quota for ugi=" + ugi + " table=" + table + " userLimiter=" + userLimiter + " tableLimiter=" + tableLimiter + " nsLimiter=" + nsLimiter + " rsLimiter=" - + rsLimiter); + + rsLimiter + " exceedThrottleQuotaEnabled=" + exceedThrottleQuotaEnabled); } if (!useNoop) { - return new DefaultOperationQuota(this.rsServices.getConfiguration(), userLimiter, - tableLimiter, nsLimiter, rsLimiter); + if (exceedThrottleQuotaEnabled) { + return new ExceedOperationQuota(this.rsServices.getConfiguration(), rsLimiter, + userLimiter, tableLimiter, nsLimiter); + } else { + return new DefaultOperationQuota(this.rsServices.getConfiguration(), userLimiter, + tableLimiter, nsLimiter, rsLimiter); + } } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java index d6a2463dc56..dcf44b84fa5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java @@ -2575,6 +2575,12 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor, requirePermission(ctx, "isRpcThrottleEnabled", Action.ADMIN); } + @Override + public void preSwitchExceedThrottleQuota(ObserverContext ctx, + boolean enable) throws IOException { + requirePermission(ctx, "switchExceedThrottleQuota", Action.ADMIN); + } + /** * Returns the active user to which authorization checks should be applied. * If we are in the context of an RPC call, the remote user is used, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncQuotaAdminApi.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncQuotaAdminApi.java index 707cc8750d9..07d436cbe52 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncQuotaAdminApi.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncQuotaAdminApi.java @@ -187,6 +187,12 @@ public class TestAsyncQuotaAdminApi extends TestAsyncAdminBase { assertEquals(true, future2.get().booleanValue()); } + @Test + public void testSwitchExceedThrottleQuota() throws Exception { + AsyncAdmin admin = ASYNC_CONN.getAdmin(); + assertEquals(false, admin.exceedThrottleQuotaSwitch(false).get().booleanValue()); + } + private void assertNumResults(int expected, final QuotaFilter filter) throws Exception { assertEquals(expected, countResults(filter)); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java index 88cd12719bb..178c6aad639 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java @@ -597,6 +597,48 @@ public class TestQuotaAdmin { testSwitchRpcThrottle(admin, false, true); } + @Test + public void testSwitchExceedThrottleQuota() throws IOException { + String regionServer = QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY; + Admin admin = TEST_UTIL.getAdmin(); + + try { + admin.exceedThrottleQuotaSwitch(true); + fail("should not come here, because can't enable exceed throttle quota " + + "if there is no region server quota"); + } catch (IOException e) { + LOG.warn("Expected exception", e); + } + + admin.setQuota(QuotaSettingsFactory.throttleRegionServer(regionServer, + ThrottleType.WRITE_NUMBER, 100, TimeUnit.SECONDS)); + try { + admin.exceedThrottleQuotaSwitch(true); + fail("should not come here, because can't enable exceed throttle quota " + + "if there is no read region server quota"); + } catch (IOException e) { + LOG.warn("Expected exception", e); + } + + admin.setQuota(QuotaSettingsFactory.throttleRegionServer(regionServer, ThrottleType.READ_NUMBER, + 20, TimeUnit.MINUTES)); + try { + admin.exceedThrottleQuotaSwitch(true); + fail("should not come here, because can't enable exceed throttle quota " + + "because not all region server quota are in seconds time unit"); + } catch (IOException e) { + LOG.warn("Expected exception", e); + } + admin.setQuota(QuotaSettingsFactory.throttleRegionServer(regionServer, ThrottleType.READ_NUMBER, + 20, TimeUnit.SECONDS)); + + assertFalse(admin.exceedThrottleQuotaSwitch(true)); + assertTrue(admin.exceedThrottleQuotaSwitch(true)); + assertTrue(admin.exceedThrottleQuotaSwitch(false)); + assertFalse(admin.exceedThrottleQuotaSwitch(false)); + admin.setQuota(QuotaSettingsFactory.unthrottleRegionServer(regionServer)); + } + private void testSwitchRpcThrottle(Admin admin, boolean oldRpcThrottle, boolean newRpcThrottle) throws IOException { boolean state = admin.switchRpcThrottle(newRpcThrottle); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java index aee97070d21..ac3dc156197 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java @@ -579,20 +579,81 @@ public class TestQuotaThrottle { // requests are throttled by table quota admin.setQuota(QuotaSettingsFactory.throttleRegionServer( QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY, ThrottleType.WRITE_NUMBER, 7, TimeUnit.MINUTES)); - triggerCacheRefresh(false, false, true, false, true, TABLE_NAMES[0]); + triggerTableCacheRefresh(false, TABLE_NAMES[0]); + triggerRegionServerCacheRefresh(false); assertEquals(5, doPuts(10, tables[0])); // requests are throttled by region server quota admin.setQuota(QuotaSettingsFactory.throttleRegionServer( QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY, ThrottleType.WRITE_NUMBER, 4, TimeUnit.MINUTES)); - triggerCacheRefresh(false, false, false, false, true, TABLE_NAMES[0]); + triggerRegionServerCacheRefresh(false); assertEquals(4, doPuts(10, tables[0])); // unthrottle admin.setQuota( QuotaSettingsFactory.unthrottleRegionServer(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY)); admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0])); - triggerCacheRefresh(true, false, true, false, true, TABLE_NAMES[0]); + triggerTableCacheRefresh(true, TABLE_NAMES[0]); + triggerRegionServerCacheRefresh(true); + } + + @Test + public void testExceedThrottleQuota() throws Exception { + final Admin admin = TEST_UTIL.getAdmin(); + admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0], ThrottleType.WRITE_NUMBER, 5, + TimeUnit.MINUTES)); + triggerTableCacheRefresh(false, TABLE_NAMES[0]); + admin.setQuota(QuotaSettingsFactory.throttleRegionServer( + QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY, ThrottleType.WRITE_NUMBER, 20, TimeUnit.SECONDS)); + admin.setQuota(QuotaSettingsFactory.throttleRegionServer( + QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY, ThrottleType.READ_NUMBER, 10, TimeUnit.SECONDS)); + triggerRegionServerCacheRefresh(false); + + // enable exceed throttle quota + admin.exceedThrottleQuotaSwitch(true); + // exceed table limit and allowed by region server limit + triggerExceedThrottleQuotaCacheRefresh(true); + waitMinuteQuota(); + assertEquals(10, doPuts(10, tables[0])); + // exceed table limit and throttled by region server limit + waitMinuteQuota(); + assertEquals(20, doPuts(25, tables[0])); + + // set region server limiter is lower than table limiter + admin.setQuota(QuotaSettingsFactory.throttleRegionServer( + QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY, ThrottleType.WRITE_NUMBER, 2, TimeUnit.SECONDS)); + triggerRegionServerCacheRefresh(false); + // throttled by region server limiter + waitMinuteQuota(); + assertEquals(2, doPuts(10, tables[0])); + admin.setQuota(QuotaSettingsFactory.throttleRegionServer( + QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY, ThrottleType.WRITE_NUMBER, 20, TimeUnit.SECONDS)); + triggerRegionServerCacheRefresh(false); + + // disable exceed throttle quota + admin.exceedThrottleQuotaSwitch(false); + triggerExceedThrottleQuotaCacheRefresh(false); + waitMinuteQuota(); + // throttled by table limit + assertEquals(5, doPuts(10, tables[0])); + + // enable exceed throttle quota and unthrottle region server + admin.exceedThrottleQuotaSwitch(true); + triggerExceedThrottleQuotaCacheRefresh(true); + waitMinuteQuota(); + admin.setQuota( + QuotaSettingsFactory.unthrottleRegionServer(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY)); + triggerRegionServerCacheRefresh(true); + waitMinuteQuota(); + // throttled by table limit + assertEquals(5, doPuts(10, tables[0])); + + // disable exceed throttle quota + admin.exceedThrottleQuotaSwitch(false); + triggerExceedThrottleQuotaCacheRefresh(false); + // unthrottle table + admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0])); + triggerTableCacheRefresh(true, TABLE_NAMES[0]); } private int doPuts(int maxOps, final Table... tables) throws Exception { @@ -647,29 +708,39 @@ public class TestQuotaThrottle { } private void triggerUserCacheRefresh(boolean bypass, TableName... tables) throws Exception { - triggerCacheRefresh(bypass, true, false, false, false, tables); + triggerCacheRefresh(bypass, true, false, false, false, false, tables); } private void triggerTableCacheRefresh(boolean bypass, TableName... tables) throws Exception { - triggerCacheRefresh(bypass, false, true, false, false, tables); + triggerCacheRefresh(bypass, false, true, false, false, false, tables); } private void triggerNamespaceCacheRefresh(boolean bypass, TableName... tables) throws Exception { - triggerCacheRefresh(bypass, false, false, true, false, tables); + triggerCacheRefresh(bypass, false, false, true, false, false, tables); + } + + private void triggerRegionServerCacheRefresh(boolean bypass) throws Exception { + triggerCacheRefresh(bypass, false, false, false, true, false); + } + + private void triggerExceedThrottleQuotaCacheRefresh(boolean exceedEnabled) throws Exception { + triggerCacheRefresh(exceedEnabled, false, false, false, false, true); } private void triggerCacheRefresh(boolean bypass, boolean userLimiter, boolean tableLimiter, - boolean nsLimiter, boolean rsLimiter, final TableName... tables) throws Exception { + boolean nsLimiter, boolean rsLimiter, boolean exceedThrottleQuota, final TableName... tables) + throws Exception { envEdge.incValue(2 * REFRESH_TIME); - for (RegionServerThread rst: TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) { - RegionServerRpcQuotaManager quotaManager = rst.getRegionServer().getRegionServerRpcQuotaManager(); + for (RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) { + RegionServerRpcQuotaManager quotaManager = + rst.getRegionServer().getRegionServerRpcQuotaManager(); QuotaCache quotaCache = quotaManager.getQuotaCache(); quotaCache.triggerCacheRefresh(); // sleep for cache update Thread.sleep(250); - for (TableName table: tables) { + for (TableName table : tables) { quotaCache.getTableLimiter(table); } @@ -677,7 +748,7 @@ public class TestQuotaThrottle { while (!isUpdated) { quotaCache.triggerCacheRefresh(); isUpdated = true; - for (TableName table: tables) { + for (TableName table : tables) { boolean isBypass = true; if (userLimiter) { isBypass &= quotaCache.getUserLimiter(User.getCurrent().getUGI(), table).isBypass(); @@ -688,17 +759,27 @@ public class TestQuotaThrottle { if (nsLimiter) { isBypass &= quotaCache.getNamespaceLimiter(table.getNamespaceAsString()).isBypass(); } - if (rsLimiter) { - isBypass &= quotaCache - .getRegionServerQuotaLimiter(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY) - .isBypass(); - } if (isBypass != bypass) { envEdge.incValue(100); isUpdated = false; break; } } + if (rsLimiter) { + boolean rsIsBypass = quotaCache + .getRegionServerQuotaLimiter(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY).isBypass(); + if (rsIsBypass != bypass) { + envEdge.incValue(100); + isUpdated = false; + continue; + } + } + if (exceedThrottleQuota) { + if (quotaCache.isExceedThrottleQuotaEnabled() != bypass) { + envEdge.incValue(100); + isUpdated = false; + } + } } LOG.debug("QuotaCache"); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java index 5199550dee8..2463eb0f8ee 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java @@ -3517,6 +3517,20 @@ public class TestAccessController extends SecureTestUtil { verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); } + @Test + public void testSwitchExceedThrottleQuota() throws Exception { + AccessTestAction action = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preSwitchExceedThrottleQuota(ObserverContextImpl.createAndPrepare(CP_ENV), + true); + return null; + } + }; + verifyAllowed(action, SUPERUSER, USER_ADMIN); + verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + } + /* * Validate Global User ACL */ diff --git a/hbase-shell/src/main/ruby/hbase/quotas.rb b/hbase-shell/src/main/ruby/hbase/quotas.rb index 4023aedc1cb..a0f2263914b 100644 --- a/hbase-shell/src/main/ruby/hbase/quotas.rb +++ b/hbase-shell/src/main/ruby/hbase/quotas.rb @@ -264,6 +264,10 @@ module Hbase @admin.switchRpcThrottle(java.lang.Boolean.valueOf(enabled)) end + def switch_exceed_throttle_quota(enabled) + @admin.exceedThrottleQuotaSwitch(java.lang.Boolean.valueOf(enabled)) + end + def _parse_size(str_limit) str_limit = str_limit.downcase match = /^(\d+)([bkmgtp%]?)$/.match(str_limit) diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index c22652d0056..a8e58ed6560 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -442,6 +442,8 @@ Shell.load_command_group( list_snapshot_sizes enable_rpc_throttle disable_rpc_throttle + enable_exceed_throttle_quota + disable_exceed_throttle_quota ] ) diff --git a/hbase-shell/src/main/ruby/shell/commands/disable_exceed_throttle_quota.rb b/hbase-shell/src/main/ruby/shell/commands/disable_exceed_throttle_quota.rb new file mode 100644 index 00000000000..2baa05a42c9 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/disable_exceed_throttle_quota.rb @@ -0,0 +1,40 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class DisableExceedThrottleQuota < Command + def help + <<-EOF +Disable exceed throttle quota. Returns previous exceed throttle quota enabled value. +NOTE: if quota is not enabled, this will not work and always return false. + +Examples: + hbase> disable_exceed_throttle_quota + EOF + end + + def command + prev_state = quotas_admin.switch_exceed_throttle_quota(false) ? 'true' : 'false' + formatter.row(["Previous exceed throttle quota enabled : #{prev_state}"]) + prev_state + end + end + end +end diff --git a/hbase-shell/src/main/ruby/shell/commands/enable_exceed_throttle_quota.rb b/hbase-shell/src/main/ruby/shell/commands/enable_exceed_throttle_quota.rb new file mode 100644 index 00000000000..5ab27c8512a --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/enable_exceed_throttle_quota.rb @@ -0,0 +1,50 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class EnableExceedThrottleQuota < Command + def help + <<-EOF +Enable exceed throttle quota. Returns previous exceed throttle quota enabled value. +NOTE: if quota is not enabled, this will not work and always return false. + +If enabled, allow requests exceed user/table/namespace throttle quotas when region +server has available quota. + +There are two limits if enable exceed throttle quota. First, please set region server +quota. Second, please make sure that all region server throttle quotas are in seconds +time unit, because once previous requests exceed their quota and consume region server +quota, quota in other time units may be refilled in a long time, which may affect later +requests. + + +Examples: + hbase> enable_exceed_throttle_quota + EOF + end + + def command + prev_state = quotas_admin.switch_exceed_throttle_quota(true) ? 'true' : 'false' + formatter.row(["Previous exceed throttle quota enabled : #{prev_state}"]) + prev_state + end + end + end +end diff --git a/hbase-shell/src/test/ruby/hbase/quotas_test.rb b/hbase-shell/src/test/ruby/hbase/quotas_test.rb index 1bcc07b7e58..2d85054d303 100644 --- a/hbase-shell/src/test/ruby/hbase/quotas_test.rb +++ b/hbase-shell/src/test/ruby/hbase/quotas_test.rb @@ -272,6 +272,16 @@ module Hbase output = capture_stdout{ command(:list_quotas) } assert(output.include?('0 row(s)')) end + + define_test 'switch exceed throttle quota' do + command(:set_quota, TYPE => THROTTLE, REGIONSERVER => 'all', LIMIT => '1CU/sec') + output = capture_stdout { command(:enable_exceed_throttle_quota) } + assert(output.include?('Previous exceed throttle quota enabled : false')) + + output = capture_stdout { command(:disable_exceed_throttle_quota) } + assert(output.include?('Previous exceed throttle quota enabled : true')) + command(:set_quota, TYPE => THROTTLE, REGIONSERVER => 'all', LIMIT => NONE) + end end # rubocop:enable Metrics/ClassLength end diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java index 5d550c6888a..4063a3c1ef1 100644 --- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java +++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java @@ -512,6 +512,12 @@ public class ThriftAdmin implements Admin { "isRpcThrottleEnabled by pattern not supported in ThriftAdmin"); } + @Override + public boolean exceedThrottleQuotaSwitch(boolean enable) throws IOException { + throw new NotImplementedException( + "exceedThrottleQuotaSwitch by pattern not supported in ThriftAdmin"); + } + @Override public HTableDescriptor[] disableTables(String regex) throws IOException { throw new NotImplementedException("disableTables by pattern not supported in ThriftAdmin");