diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaFilter.java index 30cdaf19809..d5190ad9f7c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaFilter.java @@ -33,6 +33,7 @@ public class QuotaFilter { private String namespaceRegex; private String tableRegex; private String userRegex; + private String regionServerRegex; public QuotaFilter() { } @@ -70,6 +71,17 @@ public class QuotaFilter { return this; } + /** + * Set the region server filter regex + * @param regex the region server filter + * @return the quota filter object + */ + public QuotaFilter setRegionServerFilter(final String regex) { + this.regionServerRegex = regex; + hasFilters |= StringUtils.isNotEmpty(regex); + return this; + } + /** * Add a type to the filter list * @param type the type to filter on @@ -105,4 +117,9 @@ public class QuotaFilter { public String getUserFilter() { return userRegex; } + + /** @return the RegionServer filter regex */ + public String getRegionServerFilter() { + return regionServerRegex; + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettings.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettings.java index 3351a25038b..d644753a479 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettings.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettings.java @@ -22,24 +22,26 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory.QuotaGlobalsSettingsBypass; import org.apache.yetus.audience.InterfaceAudience; import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRequest; -import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory.QuotaGlobalsSettingsBypass; @InterfaceAudience.Public public abstract class QuotaSettings { private final String userName; private final String namespace; private final TableName tableName; + private final String regionServer; - protected QuotaSettings(final String userName, final TableName tableName, - final String namespace) { + protected QuotaSettings(final String userName, final TableName tableName, final String namespace, + final String regionServer) { this.userName = userName; this.namespace = namespace; this.tableName = tableName; + this.regionServer = regionServer; } public abstract QuotaType getQuotaType(); @@ -56,6 +58,10 @@ public abstract class QuotaSettings { return namespace; } + public String getRegionServer() { + return regionServer; + } + /** * Converts the protocol buffer request into a QuotaSetting POJO. Arbitrarily * enforces that the request only contain one "limit", despite the message @@ -78,6 +84,10 @@ public abstract class QuotaSettings { if (request.hasNamespace()) { namespace = request.getNamespace(); } + String regionServer = null; + if (request.hasRegionServer()) { + regionServer = request.getRegionServer(); + } if (request.hasBypassGlobals()) { // Make sure we don't have either of the two below limits also included if (request.hasSpaceLimit() || request.hasThrottle()) { @@ -85,7 +95,7 @@ public abstract class QuotaSettings { "SetQuotaRequest has multiple limits: " + TextFormat.shortDebugString(request)); } return new QuotaGlobalsSettingsBypass( - username, tableName, namespace, request.getBypassGlobals()); + username, tableName, namespace, regionServer, request.getBypassGlobals()); } else if (request.hasSpaceLimit()) { // Make sure we don't have the below limit as well if (request.hasThrottle()) { @@ -100,7 +110,8 @@ public abstract class QuotaSettings { return QuotaSettingsFactory.fromSpace( tableName, namespace, request.getSpaceLimit().getQuota()); } else if (request.hasThrottle()) { - return new ThrottleSettings(username, tableName, namespace, request.getThrottle()); + return new ThrottleSettings(username, tableName, namespace, regionServer, + request.getThrottle()); } else { throw new IllegalStateException("Unhandled SetRequestRequest state"); } @@ -123,6 +134,9 @@ public abstract class QuotaSettings { if (settings.getNamespace() != null) { builder.setNamespace(settings.getNamespace()); } + if (settings.getRegionServer() != null) { + builder.setRegionServer(settings.getRegionServer()); + } settings.setupSetQuotaRequest(builder); return builder.build(); } @@ -152,6 +166,9 @@ public abstract class QuotaSettings { builder.append(namespace); builder.append("', "); } + if (regionServer != null) { + builder.append("REGIONSERVER => ").append(regionServer).append(", "); + } return builder.toString(); } @@ -203,5 +220,8 @@ public abstract class QuotaSettings { if (!Objects.equals(getNamespace(), mergee.getNamespace())) { throw new IllegalArgumentException("Mismatched namespace on settings to merge"); } + if (!Objects.equals(getRegionServer(), mergee.getRegionServer())) { + throw new IllegalArgumentException("Mismatched region server on settings to merge"); + } } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java index 14d1ad31b3d..48f7f65f2cd 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java @@ -25,8 +25,8 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.hbase.TableName; import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRequest; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota; @@ -37,8 +37,8 @@ public class QuotaSettingsFactory { private final boolean bypassGlobals; QuotaGlobalsSettingsBypass(final String userName, final TableName tableName, - final String namespace, final boolean bypassGlobals) { - super(userName, tableName, namespace); + final String namespace, final String regionServer, final boolean bypassGlobals) { + super(userName, tableName, namespace, regionServer); this.bypassGlobals = bypassGlobals; } @@ -80,35 +80,42 @@ public class QuotaSettingsFactory { * QuotaSettings from the Quotas object */ static List fromUserQuotas(final String userName, final Quotas quotas) { - return fromQuotas(userName, null, null, quotas); + return fromQuotas(userName, null, null, null, quotas); } static List fromUserQuotas(final String userName, final TableName tableName, final Quotas quotas) { - return fromQuotas(userName, tableName, null, quotas); + return fromQuotas(userName, tableName, null, null, quotas); } static List fromUserQuotas(final String userName, final String namespace, final Quotas quotas) { - return fromQuotas(userName, null, namespace, quotas); + return fromQuotas(userName, null, namespace, null, quotas); } static List fromTableQuotas(final TableName tableName, final Quotas quotas) { - return fromQuotas(null, tableName, null, quotas); + return fromQuotas(null, tableName, null, null, quotas); } static List fromNamespaceQuotas(final String namespace, final Quotas quotas) { - return fromQuotas(null, null, namespace, quotas); + return fromQuotas(null, null, namespace, null, quotas); + } + + static List fromRegionServerQuotas(final String regionServer, + final Quotas quotas) { + return fromQuotas(null, null, null, regionServer, quotas); } private static List fromQuotas(final String userName, final TableName tableName, - final String namespace, final Quotas quotas) { + final String namespace, final String regionServer, final Quotas quotas) { List settings = new ArrayList<>(); if (quotas.hasThrottle()) { - settings.addAll(fromThrottle(userName, tableName, namespace, quotas.getThrottle())); + settings + .addAll(fromThrottle(userName, tableName, namespace, regionServer, quotas.getThrottle())); } if (quotas.getBypassGlobals() == true) { - settings.add(new QuotaGlobalsSettingsBypass(userName, tableName, namespace, true)); + settings + .add(new QuotaGlobalsSettingsBypass(userName, tableName, namespace, regionServer, true)); } if (quotas.hasSpace()) { settings.add(fromSpace(tableName, namespace, quotas.getSpace())); @@ -116,43 +123,44 @@ public class QuotaSettingsFactory { return settings; } - protected static List fromThrottle(final String userName, final TableName tableName, - final String namespace, final QuotaProtos.Throttle throttle) { + protected static List fromThrottle(final String userName, + final TableName tableName, final String namespace, final String regionServer, + final QuotaProtos.Throttle throttle) { List settings = new ArrayList<>(); if (throttle.hasReqNum()) { - settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, - ThrottleType.REQUEST_NUMBER, throttle.getReqNum())); + settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, regionServer, + ThrottleType.REQUEST_NUMBER, throttle.getReqNum())); } if (throttle.hasReqSize()) { - settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, - ThrottleType.REQUEST_SIZE, throttle.getReqSize())); + settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, regionServer, + ThrottleType.REQUEST_SIZE, throttle.getReqSize())); } if (throttle.hasWriteNum()) { - settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, - ThrottleType.WRITE_NUMBER, throttle.getWriteNum())); + settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, regionServer, + ThrottleType.WRITE_NUMBER, throttle.getWriteNum())); } if (throttle.hasWriteSize()) { - settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, - ThrottleType.WRITE_SIZE, throttle.getWriteSize())); + settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, regionServer, + ThrottleType.WRITE_SIZE, throttle.getWriteSize())); } if (throttle.hasReadNum()) { - settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, - ThrottleType.READ_NUMBER, throttle.getReadNum())); + settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, regionServer, + ThrottleType.READ_NUMBER, throttle.getReadNum())); } if (throttle.hasReadSize()) { - settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, - ThrottleType.READ_SIZE, throttle.getReadSize())); + settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, regionServer, + ThrottleType.READ_SIZE, throttle.getReadSize())); } if (throttle.hasReqCapacityUnit()) { - settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, + settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, regionServer, ThrottleType.REQUEST_CAPACITY_UNIT, throttle.getReqCapacityUnit())); } if (throttle.hasReadCapacityUnit()) { - settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, + settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, regionServer, ThrottleType.READ_CAPACITY_UNIT, throttle.getReadCapacityUnit())); } if (throttle.hasWriteCapacityUnit()) { - settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, + settings.add(ThrottleSettings.fromTimedQuota(userName, tableName, namespace, regionServer, ThrottleType.WRITE_CAPACITY_UNIT, throttle.getWriteCapacityUnit())); } return settings; @@ -195,7 +203,7 @@ public class QuotaSettingsFactory { */ public static QuotaSettings throttleUser(final String userName, final ThrottleType type, final long limit, final TimeUnit timeUnit) { - return throttle(userName, null, null, type, limit, timeUnit); + return throttle(userName, null, null, null, type, limit, timeUnit); } /** @@ -210,7 +218,7 @@ public class QuotaSettingsFactory { */ public static QuotaSettings throttleUser(final String userName, final TableName tableName, final ThrottleType type, final long limit, final TimeUnit timeUnit) { - return throttle(userName, tableName, null, type, limit, timeUnit); + return throttle(userName, tableName, null, null, type, limit, timeUnit); } /** @@ -225,7 +233,7 @@ public class QuotaSettingsFactory { */ public static QuotaSettings throttleUser(final String userName, final String namespace, final ThrottleType type, final long limit, final TimeUnit timeUnit) { - return throttle(userName, null, namespace, type, limit, timeUnit); + return throttle(userName, null, namespace, null, type, limit, timeUnit); } /** @@ -235,7 +243,7 @@ public class QuotaSettingsFactory { * @return the quota settings */ public static QuotaSettings unthrottleUser(final String userName) { - return throttle(userName, null, null, null, 0, null); + return throttle(userName, null, null, null, null, 0, null); } /** @@ -246,7 +254,7 @@ public class QuotaSettingsFactory { * @return the quota settings */ public static QuotaSettings unthrottleUser(final String userName, final TableName tableName) { - return throttle(userName, tableName, null, null, 0, null); + return throttle(userName, tableName, null, null, null, 0, null); } /** @@ -257,7 +265,7 @@ public class QuotaSettingsFactory { * @return the quota settings */ public static QuotaSettings unthrottleUser(final String userName, final String namespace) { - return throttle(userName, null, namespace, null, 0, null); + return throttle(userName, null, namespace, null, null, 0, null); } /** @@ -271,7 +279,7 @@ public class QuotaSettingsFactory { */ public static QuotaSettings throttleTable(final TableName tableName, final ThrottleType type, final long limit, final TimeUnit timeUnit) { - return throttle(null, tableName, null, type, limit, timeUnit); + return throttle(null, tableName, null, null, type, limit, timeUnit); } /** @@ -281,7 +289,7 @@ public class QuotaSettingsFactory { * @return the quota settings */ public static QuotaSettings unthrottleTable(final TableName tableName) { - return throttle(null, tableName, null, null, 0, null); + return throttle(null, tableName, null, null, null, 0, null); } /** @@ -295,7 +303,7 @@ public class QuotaSettingsFactory { */ public static QuotaSettings throttleNamespace(final String namespace, final ThrottleType type, final long limit, final TimeUnit timeUnit) { - return throttle(null, null, namespace, type, limit, timeUnit); + return throttle(null, null, namespace, null, type, limit, timeUnit); } /** @@ -305,12 +313,36 @@ public class QuotaSettingsFactory { * @return the quota settings */ public static QuotaSettings unthrottleNamespace(final String namespace) { - return throttle(null, null, namespace, null, 0, null); + return throttle(null, null, namespace, null, null, 0, null); + } + + /** + * Throttle the specified region server. + * + * @param regionServer the region server to throttle + * @param type the type of throttling + * @param limit the allowed number of request/data per timeUnit + * @param timeUnit the limit time unit + * @return the quota settings + */ + public static QuotaSettings throttleRegionServer(final String regionServer, + final ThrottleType type, final long limit, final TimeUnit timeUnit) { + return throttle(null, null, null, regionServer, type, limit, timeUnit); + } + + /** + * Remove the throttling for the specified region server. + * + * @param regionServer the region Server + * @return the quota settings + */ + public static QuotaSettings unthrottleRegionServer(final String regionServer) { + return throttle(null, null, null, regionServer, null, 0, null); } /* Throttle helper */ private static QuotaSettings throttle(final String userName, final TableName tableName, - final String namespace, final ThrottleType type, final long limit, + final String namespace, final String regionServer, final ThrottleType type, final long limit, final TimeUnit timeUnit) { QuotaProtos.ThrottleRequest.Builder builder = QuotaProtos.ThrottleRequest.newBuilder(); if (type != null) { @@ -319,7 +351,7 @@ public class QuotaSettingsFactory { if (timeUnit != null) { builder.setTimedQuota(ProtobufUtil.toTimedQuota(limit, timeUnit, QuotaScope.MACHINE)); } - return new ThrottleSettings(userName, tableName, namespace, builder.build()); + return new ThrottleSettings(userName, tableName, namespace, regionServer, builder.build()); } /* ========================================================================== @@ -334,7 +366,7 @@ public class QuotaSettingsFactory { * @return the quota settings */ public static QuotaSettings bypassGlobals(final String userName, final boolean bypassGlobals) { - return new QuotaGlobalsSettingsBypass(userName, null, null, bypassGlobals); + return new QuotaGlobalsSettingsBypass(userName, null, null, null, bypassGlobals); } /* ========================================================================== diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java index cebb9e2a505..42ab58f14cb 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java @@ -99,6 +99,13 @@ public class QuotaTableUtil { protected static final byte[] QUOTA_USER_ROW_KEY_PREFIX = Bytes.toBytes("u."); protected static final byte[] QUOTA_TABLE_ROW_KEY_PREFIX = Bytes.toBytes("t."); protected static final byte[] QUOTA_NAMESPACE_ROW_KEY_PREFIX = Bytes.toBytes("n."); + protected static final byte[] QUOTA_REGION_SERVER_ROW_KEY_PREFIX = Bytes.toBytes("r."); + + /* + * TODO: Setting specified region server quota isn't supported currently and the row key "r.all" + * represents the throttle quota of all region servers + */ + public static final String QUOTA_REGION_SERVER_ROW_KEY = "all"; /* ========================================================================= * Quota "settings" helpers @@ -134,6 +141,11 @@ public class QuotaTableUtil { return getQuotas(connection, rowKey, QUOTA_QUALIFIER_SETTINGS); } + public static Quotas getRegionServerQuota(final Connection connection, final String regionServer) + throws IOException { + return getQuotas(connection, getRegionServerRowKey(regionServer)); + } + private static Quotas getQuotas(final Connection connection, final byte[] rowKey, final byte[] qualifier) throws IOException { Get get = new Get(rowKey); @@ -157,6 +169,12 @@ public class QuotaTableUtil { return get; } + public static Get makeGetForRegionServerQuotas(final String regionServer) { + Get get = new Get(getRegionServerRowKey(regionServer)); + get.addFamily(QUOTA_FAMILY_INFO); + return get; + } + public static Get makeGetForUserQuotas(final String user, final Iterable tables, final Iterable namespaces) { Get get = new Get(getUserRowKey(user)); @@ -220,6 +238,9 @@ public class QuotaTableUtil { } else if (StringUtils.isNotEmpty(filter.getNamespaceFilter())) { filterList.addFilter(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(getNamespaceRowKeyRegex(filter.getNamespaceFilter()), 0))); + } else if (StringUtils.isNotEmpty(filter.getRegionServerFilter())) { + filterList.addFilter(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator( + getRegionServerRowKeyRegex(filter.getRegionServerFilter()), 0))); } return filterList; } @@ -318,8 +339,13 @@ public class QuotaTableUtil { throws IOException; } - public static interface QuotasVisitor extends UserQuotasVisitor, - TableQuotasVisitor, NamespaceQuotasVisitor { + private static interface RegionServerQuotasVisitor { + void visitRegionServerQuotas(final String regionServer, final Quotas quotas) + throws IOException; + } + + public static interface QuotasVisitor extends UserQuotasVisitor, TableQuotasVisitor, + NamespaceQuotasVisitor, RegionServerQuotasVisitor { } public static void parseResult(final Result result, final QuotasVisitor visitor) @@ -331,6 +357,8 @@ public class QuotaTableUtil { parseTableResult(result, visitor); } else if (isUserRowKey(row)) { parseUserResult(result, visitor); + } else if (isRegionServerRowKey(row)) { + parseRegionServerResult(result, visitor); } else { LOG.warn("unexpected row-key: " + Bytes.toString(row)); } @@ -364,6 +392,11 @@ public class QuotaTableUtil { public void visitNamespaceQuotas(String namespace, Quotas quotas) { quotaSettings.addAll(QuotaSettingsFactory.fromNamespaceQuotas(namespace, quotas)); } + + @Override + public void visitRegionServerQuotas(String regionServer, Quotas quotas) { + quotaSettings.addAll(QuotaSettingsFactory.fromRegionServerQuotas(regionServer, quotas)); + } }); } @@ -382,6 +415,21 @@ public class QuotaTableUtil { } } + private static void parseRegionServerResult(final Result result, + final RegionServerQuotasVisitor visitor) throws IOException { + String rs = getRegionServerFromRowKey(result.getRow()); + parseRegionServerResult(rs, result, visitor); + } + + private static void parseRegionServerResult(final String regionServer, final Result result, + final RegionServerQuotasVisitor visitor) throws IOException { + byte[] data = result.getValue(QUOTA_FAMILY_INFO, QUOTA_QUALIFIER_SETTINGS); + if (data != null) { + Quotas quotas = quotasFromData(data); + visitor.visitRegionServerQuotas(regionServer, quotas); + } + } + public static void parseTableResult(final Result result, final TableQuotasVisitor visitor) throws IOException { TableName table = getTableFromRowKey(result.getRow()); @@ -621,6 +669,10 @@ public class QuotaTableUtil { return Bytes.add(QUOTA_NAMESPACE_ROW_KEY_PREFIX, Bytes.toBytes(namespace)); } + protected static byte[] getRegionServerRowKey(final String regionServer) { + return Bytes.add(QUOTA_REGION_SERVER_ROW_KEY_PREFIX, Bytes.toBytes(regionServer)); + } + protected static byte[] getSettingsQualifierForUserTable(final TableName tableName) { return Bytes.add(QUOTA_QUALIFIER_SETTINGS_PREFIX, tableName.getName()); } @@ -642,6 +694,10 @@ public class QuotaTableUtil { return getRowKeyRegEx(QUOTA_NAMESPACE_ROW_KEY_PREFIX, namespace); } + private static String getRegionServerRowKeyRegex(final String regionServer) { + return getRowKeyRegEx(QUOTA_REGION_SERVER_ROW_KEY_PREFIX, regionServer); + } + private static String getRowKeyRegEx(final byte[] prefix, final String regex) { return '^' + Pattern.quote(Bytes.toString(prefix)) + regex + '$'; } @@ -664,6 +720,14 @@ public class QuotaTableUtil { return Bytes.toString(key, QUOTA_NAMESPACE_ROW_KEY_PREFIX.length); } + protected static boolean isRegionServerRowKey(final byte[] key) { + return Bytes.startsWith(key, QUOTA_REGION_SERVER_ROW_KEY_PREFIX); + } + + protected static String getRegionServerFromRowKey(final byte[] key) { + return Bytes.toString(key, QUOTA_REGION_SERVER_ROW_KEY_PREFIX.length); + } + protected static boolean isTableRowKey(final byte[] key) { return Bytes.startsWith(key, QUOTA_TABLE_ROW_KEY_PREFIX); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java index 69baf62206a..6f754ba1027 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java @@ -37,7 +37,7 @@ class SpaceLimitSettings extends QuotaSettings { private final SpaceLimitRequest proto; SpaceLimitSettings(TableName tableName, long sizeLimit, SpaceViolationPolicy violationPolicy) { - super(null, Objects.requireNonNull(tableName), null); + super(null, Objects.requireNonNull(tableName), null, null); validateSizeLimit(sizeLimit); proto = buildProtoAddQuota(sizeLimit, Objects.requireNonNull(violationPolicy)); } @@ -46,12 +46,12 @@ class SpaceLimitSettings extends QuotaSettings { * Constructs a {@code SpaceLimitSettings} to remove a space quota on the given {@code tableName}. */ SpaceLimitSettings(TableName tableName) { - super(null, Objects.requireNonNull(tableName), null); + super(null, Objects.requireNonNull(tableName), null, null); proto = buildProtoRemoveQuota(); } SpaceLimitSettings(String namespace, long sizeLimit, SpaceViolationPolicy violationPolicy) { - super(null, null, Objects.requireNonNull(namespace)); + super(null, null, Objects.requireNonNull(namespace), null); validateSizeLimit(sizeLimit); proto = buildProtoAddQuota(sizeLimit, Objects.requireNonNull(violationPolicy)); } @@ -60,12 +60,12 @@ class SpaceLimitSettings extends QuotaSettings { * Constructs a {@code SpaceLimitSettings} to remove a space quota on the given {@code namespace}. */ SpaceLimitSettings(String namespace) { - super(null, null, Objects.requireNonNull(namespace)); + super(null, null, Objects.requireNonNull(namespace), null); proto = buildProtoRemoveQuota(); } SpaceLimitSettings(TableName tableName, String namespace, SpaceLimitRequest req) { - super(null, tableName, namespace); + super(null, tableName, namespace, null); proto = req; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottleSettings.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottleSettings.java index 05fb70bfd50..c6166207b15 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottleSettings.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottleSettings.java @@ -25,19 +25,19 @@ import org.apache.hadoop.hbase.TableName; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRequest; -import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.TimedQuota; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.TimedQuota; @InterfaceAudience.Private @InterfaceStability.Evolving class ThrottleSettings extends QuotaSettings { final QuotaProtos.ThrottleRequest proto; - ThrottleSettings(final String userName, final TableName tableName, - final String namespace, final QuotaProtos.ThrottleRequest proto) { - super(userName, tableName, namespace); + ThrottleSettings(final String userName, final TableName tableName, final String namespace, + final String regionServer, final QuotaProtos.ThrottleRequest proto) { + super(userName, tableName, namespace, regionServer); this.proto = proto; } @@ -146,7 +146,8 @@ class ThrottleSettings extends QuotaSettings { QuotaProtos.ThrottleRequest mergedReq = builder.setTimedQuota( timedQuotaBuilder.build()).build(); - return new ThrottleSettings(getUserName(), getTableName(), getNamespace(), mergedReq); + return new ThrottleSettings(getUserName(), getTableName(), getNamespace(), + getRegionServer(), mergedReq); } } return this; @@ -159,12 +160,12 @@ class ThrottleSettings extends QuotaSettings { } } - static ThrottleSettings fromTimedQuota(final String userName, - final TableName tableName, final String namespace, - ThrottleType type, QuotaProtos.TimedQuota timedQuota) { + static ThrottleSettings fromTimedQuota(final String userName, final TableName tableName, + final String namespace, final String regionServer, ThrottleType type, + QuotaProtos.TimedQuota timedQuota) { QuotaProtos.ThrottleRequest.Builder builder = QuotaProtos.ThrottleRequest.newBuilder(); builder.setType(ProtobufUtil.toProtoThrottleType(type)); builder.setTimedQuota(timedQuota); - return new ThrottleSettings(userName, tableName, namespace, builder.build()); + return new ThrottleSettings(userName, tableName, namespace, regionServer, builder.build()); } } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaGlobalsSettingsBypass.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaGlobalsSettingsBypass.java index 5b716f1c796..37a21dc2b18 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaGlobalsSettingsBypass.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaGlobalsSettingsBypass.java @@ -39,67 +39,103 @@ public class TestQuotaGlobalsSettingsBypass { @Test public void testMerge() throws IOException { - QuotaGlobalsSettingsBypass orig = new QuotaGlobalsSettingsBypass("joe", null, null, true); - assertFalse(orig.merge(new QuotaGlobalsSettingsBypass( - "joe", null, null, false)).getBypass()); + QuotaGlobalsSettingsBypass orig = new QuotaGlobalsSettingsBypass("joe", null, null, null, true); + assertFalse( + orig.merge(new QuotaGlobalsSettingsBypass("joe", null, null, null, false)).getBypass()); } @Test public void testInvalidMerges() throws IOException { - QuotaGlobalsSettingsBypass userBypass = new QuotaGlobalsSettingsBypass( - "joe", null, null, true); - QuotaGlobalsSettingsBypass tableBypass = new QuotaGlobalsSettingsBypass( - null, TableName.valueOf("table"), null, true); - QuotaGlobalsSettingsBypass namespaceBypass = new QuotaGlobalsSettingsBypass( - null, null, "ns", true); - QuotaGlobalsSettingsBypass userOnTableBypass = new QuotaGlobalsSettingsBypass( - "joe", TableName.valueOf("table"), null, true); - QuotaGlobalsSettingsBypass userOnNamespaceBypass = new QuotaGlobalsSettingsBypass( - "joe", null, "ns", true); + QuotaGlobalsSettingsBypass userBypass = + new QuotaGlobalsSettingsBypass("joe", null, null, null, true); + QuotaGlobalsSettingsBypass tableBypass = + new QuotaGlobalsSettingsBypass(null, TableName.valueOf("table"), null, null, true); + QuotaGlobalsSettingsBypass namespaceBypass = + new QuotaGlobalsSettingsBypass(null, null, "ns", null, true); + QuotaGlobalsSettingsBypass regionServerBypass = + new QuotaGlobalsSettingsBypass(null, null, null, "all", true); + QuotaGlobalsSettingsBypass userOnTableBypass = + new QuotaGlobalsSettingsBypass("joe", TableName.valueOf("table"), null, null, true); + QuotaGlobalsSettingsBypass userOnNamespaceBypass = + new QuotaGlobalsSettingsBypass("joe", null, "ns", null, true); + QuotaGlobalsSettingsBypass userOnRegionServerBypass = + new QuotaGlobalsSettingsBypass("joe", null, null, "all", true); assertTrue(userBypass.merge(userBypass).getBypass()); - expectFailure(userBypass, new QuotaGlobalsSettingsBypass("frank", null, null, false)); + expectFailure(userBypass, new QuotaGlobalsSettingsBypass("frank", null, null, null, false)); expectFailure(userBypass, tableBypass); expectFailure(userBypass, namespaceBypass); + expectFailure(userBypass, regionServerBypass); expectFailure(userBypass, userOnTableBypass); expectFailure(userBypass, userOnNamespaceBypass); + expectFailure(userBypass, userOnRegionServerBypass); assertTrue(tableBypass.merge(tableBypass).getBypass()); expectFailure(tableBypass, userBypass); - expectFailure(tableBypass, new QuotaGlobalsSettingsBypass( - null, TableName.valueOf("foo"), null, false)); + expectFailure(tableBypass, + new QuotaGlobalsSettingsBypass(null, TableName.valueOf("foo"), null, null, false)); expectFailure(tableBypass, namespaceBypass); + expectFailure(tableBypass, regionServerBypass); expectFailure(tableBypass, userOnTableBypass); expectFailure(tableBypass, userOnNamespaceBypass); + expectFailure(tableBypass, userOnRegionServerBypass); assertTrue(namespaceBypass.merge(namespaceBypass).getBypass()); expectFailure(namespaceBypass, userBypass); expectFailure(namespaceBypass, tableBypass); - expectFailure(namespaceBypass, new QuotaGlobalsSettingsBypass(null, null, "sn", false)); + expectFailure(namespaceBypass, regionServerBypass); + expectFailure(namespaceBypass, new QuotaGlobalsSettingsBypass(null, null, "sn", null, false)); expectFailure(namespaceBypass, userOnTableBypass); expectFailure(namespaceBypass, userOnNamespaceBypass); + expectFailure(namespaceBypass, userOnNamespaceBypass); + + assertTrue(regionServerBypass.merge(regionServerBypass).getBypass()); + expectFailure(regionServerBypass, userBypass); + expectFailure(regionServerBypass, tableBypass); + expectFailure(regionServerBypass, namespaceBypass); + expectFailure(regionServerBypass, + new QuotaGlobalsSettingsBypass(null, null, null, "rs", false)); + expectFailure(regionServerBypass, userOnTableBypass); + expectFailure(regionServerBypass, userOnNamespaceBypass); + expectFailure(regionServerBypass, userOnRegionServerBypass); assertTrue(userOnTableBypass.merge(userOnTableBypass).getBypass()); expectFailure(userOnTableBypass, userBypass); expectFailure(userOnTableBypass, tableBypass); expectFailure(userOnTableBypass, namespaceBypass); + expectFailure(userOnTableBypass, regionServerBypass); // Incorrect user - expectFailure(userOnTableBypass, new QuotaGlobalsSettingsBypass( - "frank", TableName.valueOf("foo"), null, false)); + expectFailure(userOnTableBypass, + new QuotaGlobalsSettingsBypass("frank", TableName.valueOf("foo"), null, null, false)); // Incorrect tablename - expectFailure(userOnTableBypass, new QuotaGlobalsSettingsBypass( - "joe", TableName.valueOf("bar"), null, false)); + expectFailure(userOnTableBypass, + new QuotaGlobalsSettingsBypass("joe", TableName.valueOf("bar"), null, null, false)); expectFailure(userOnTableBypass, userOnNamespaceBypass); + expectFailure(userOnTableBypass, userOnRegionServerBypass); assertTrue(userOnNamespaceBypass.merge(userOnNamespaceBypass).getBypass()); expectFailure(userOnNamespaceBypass, userBypass); expectFailure(userOnNamespaceBypass, tableBypass); expectFailure(userOnNamespaceBypass, namespaceBypass); + expectFailure(userOnNamespaceBypass, regionServerBypass); expectFailure(userOnNamespaceBypass, userOnTableBypass); - expectFailure(userOnNamespaceBypass, new QuotaGlobalsSettingsBypass( - "frank", null, "ns", false)); - expectFailure(userOnNamespaceBypass, new QuotaGlobalsSettingsBypass( - "joe", null, "sn", false)); + expectFailure(userOnNamespaceBypass, + new QuotaGlobalsSettingsBypass("frank", null, "ns", null, false)); + expectFailure(userOnNamespaceBypass, + new QuotaGlobalsSettingsBypass("joe", null, "sn", null, false)); + expectFailure(userOnNamespaceBypass, userOnRegionServerBypass); + + assertTrue(userOnRegionServerBypass.merge(userOnRegionServerBypass).getBypass()); + expectFailure(userOnRegionServerBypass, userBypass); + expectFailure(userOnRegionServerBypass, tableBypass); + expectFailure(userOnRegionServerBypass, namespaceBypass); + expectFailure(userOnRegionServerBypass, regionServerBypass); + expectFailure(userOnRegionServerBypass, userOnTableBypass); + expectFailure(userOnRegionServerBypass, userOnNamespaceBypass); + expectFailure(userOnRegionServerBypass, + new QuotaGlobalsSettingsBypass("frank", null, null, "all", false)); + expectFailure(userOnRegionServerBypass, + new QuotaGlobalsSettingsBypass("joe", null, null, "rs", false)); } void expectFailure(QuotaSettings one, QuotaSettings two) throws IOException { diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java index 2f1942c5be2..255f524dd4e 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java @@ -87,6 +87,7 @@ public class TestQuotaSettingsFactory { assertEquals(tn, throttleSettings.getTableName()); assertNull("Username should be null", throttleSettings.getUserName()); assertNull("Namespace should be null", throttleSettings.getNamespace()); + assertNull("RegionServer should be null", throttleSettings.getRegionServer()); seenRead = true; break; case WRITE_NUMBER: @@ -96,6 +97,7 @@ public class TestQuotaSettingsFactory { assertEquals(tn, throttleSettings.getTableName()); assertNull("Username should be null", throttleSettings.getUserName()); assertNull("Namespace should be null", throttleSettings.getNamespace()); + assertNull("RegionServer should be null", throttleSettings.getRegionServer()); seenWrite = true; break; default: @@ -107,6 +109,7 @@ public class TestQuotaSettingsFactory { assertEquals(tn, spaceLimit.getTableName()); assertNull("Username should be null", spaceLimit.getUserName()); assertNull("Namespace should be null", spaceLimit.getNamespace()); + assertNull("RegionServer should be null", spaceLimit.getRegionServer()); assertTrue("SpaceLimitSettings should have a SpaceQuota", spaceLimit.getProto().hasQuota()); assertEquals(spaceQuota, spaceLimit.getProto().getQuota()); seenSpace = true; diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestThrottleSettings.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestThrottleSettings.java index 91241ae7ebf..53fb9bd3e92 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestThrottleSettings.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestThrottleSettings.java @@ -48,7 +48,7 @@ public class TestThrottleSettings { .setTimeUnit(HBaseProtos.TimeUnit.MINUTES).build(); ThrottleRequest tr1 = ThrottleRequest.newBuilder().setTimedQuota(tq1) .setType(QuotaProtos.ThrottleType.REQUEST_NUMBER).build(); - ThrottleSettings orig = new ThrottleSettings("joe", null, null, tr1); + ThrottleSettings orig = new ThrottleSettings("joe", null, null, null, tr1); TimedQuota tq2 = TimedQuota.newBuilder().setSoftLimit(10) .setScope(QuotaProtos.QuotaScope.MACHINE) @@ -56,7 +56,7 @@ public class TestThrottleSettings { ThrottleRequest tr2 = ThrottleRequest.newBuilder().setTimedQuota(tq2) .setType(QuotaProtos.ThrottleType.REQUEST_NUMBER).build(); - ThrottleSettings merged = orig.merge(new ThrottleSettings("joe", null, null, tr2)); + ThrottleSettings merged = orig.merge(new ThrottleSettings("joe", null, null, null, tr2)); assertEquals(10, merged.getSoftLimit()); assertEquals(ThrottleType.REQUEST_NUMBER, merged.getThrottleType()); @@ -70,7 +70,7 @@ public class TestThrottleSettings { .setTimeUnit(HBaseProtos.TimeUnit.MINUTES).build(); ThrottleRequest requestsQuotaReq = ThrottleRequest.newBuilder().setTimedQuota(requestsQuota) .setType(QuotaProtos.ThrottleType.REQUEST_NUMBER).build(); - ThrottleSettings orig = new ThrottleSettings("joe", null, null, requestsQuotaReq); + ThrottleSettings orig = new ThrottleSettings("joe", null, null, null, requestsQuotaReq); TimedQuota readsQuota = TimedQuota.newBuilder().setSoftLimit(10) .setScope(QuotaProtos.QuotaScope.MACHINE) @@ -79,7 +79,7 @@ public class TestThrottleSettings { .setType(QuotaProtos.ThrottleType.READ_NUMBER).build(); try { - orig.merge(new ThrottleSettings("joe", null, null, readsQuotaReq)); + orig.merge(new ThrottleSettings("joe", null, null, null, readsQuotaReq)); fail("A read throttle should not be capable of being merged with a request quota"); } catch (IllegalArgumentException e) { // Pass @@ -93,13 +93,13 @@ public class TestThrottleSettings { .setTimeUnit(HBaseProtos.TimeUnit.MINUTES).build(); ThrottleRequest tr1 = ThrottleRequest.newBuilder().setTimedQuota(tq1) .setType(QuotaProtos.ThrottleType.REQUEST_NUMBER).build(); - ThrottleSettings orig = new ThrottleSettings("joe", null, null, tr1); + ThrottleSettings orig = new ThrottleSettings("joe", null, null, null, tr1); ThrottleRequest tr2 = ThrottleRequest.newBuilder() .setType(QuotaProtos.ThrottleType.REQUEST_NUMBER).build(); assertTrue( "The same object should be returned by merge, but it wasn't", - orig == orig.merge(new ThrottleSettings("joe", null, null, tr2))); + orig == orig.merge(new ThrottleSettings("joe", null, null, null, tr2))); } } diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto index 5cb4309cc62..b1ed94f55d7 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto @@ -572,6 +572,7 @@ message SetQuotaRequest { optional ThrottleRequest throttle = 7; optional SpaceLimitRequest space_limit = 8; + optional string region_server = 9; } message SetQuotaResponse { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java index 5d43f10a8b0..63ed125648f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java @@ -1092,6 +1092,24 @@ public interface MasterObserver { default void postSetNamespaceQuota(final ObserverContext ctx, final String namespace, final GlobalQuotaSettings quotas) throws IOException {} + /** + * Called before the quota for the region server is stored. + * @param ctx the environment to interact with the framework and master + * @param regionServer the name of the region server + * @param quotas the current quota for the region server + */ + default void preSetRegionServerQuota(final ObserverContext ctx, + final String regionServer, final GlobalQuotaSettings quotas) throws IOException {} + + /** + * Called after the quota for the region server is stored. + * @param ctx the environment to interact with the framework and master + * @param regionServer the name of the region server + * @param quotas the resulting quota for the region server + */ + default void postSetRegionServerQuota(final ObserverContext ctx, + final String regionServer, final GlobalQuotaSettings quotas) throws IOException {} + /** * Called before merge regions request. * @param ctx coprocessor environment diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java index dd02a36347c..b4a4421dc60 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java @@ -1285,6 +1285,26 @@ public class MasterCoprocessorHost }); } + public void preSetRegionServerQuota(final String regionServer, final GlobalQuotaSettings quotas) + throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { + @Override + public void call(MasterObserver observer) throws IOException { + observer.preSetRegionServerQuota(this, regionServer, quotas); + } + }); + } + + public void postSetRegionServerQuota(final String regionServer, final GlobalQuotaSettings quotas) + throws IOException { + execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { + @Override + public void call(MasterObserver observer) throws IOException { + observer.postSetRegionServerQuota(this, regionServer, quotas); + } + }); + } + public void preMoveServersAndTables(final Set
servers, final Set tables, final String targetGroup) throws IOException { execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/GlobalQuotaSettings.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/GlobalQuotaSettings.java index 23dc7d52ec3..b722e84bbc6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/GlobalQuotaSettings.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/GlobalQuotaSettings.java @@ -35,8 +35,9 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; @InterfaceStability.Evolving public abstract class GlobalQuotaSettings extends QuotaSettings { - protected GlobalQuotaSettings(String userName, TableName tableName, String namespace) { - super(userName, tableName, namespace); + protected GlobalQuotaSettings(String userName, TableName tableName, String namespace, + String regionServer) { + super(userName, tableName, namespace, regionServer); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/GlobalQuotaSettingsImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/GlobalQuotaSettingsImpl.java index 78f52c48e90..c7df789c200 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/GlobalQuotaSettingsImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/GlobalQuotaSettingsImpl.java @@ -27,13 +27,14 @@ import java.util.Map.Entry; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory.QuotaGlobalsSettingsBypass; +import org.apache.yetus.audience.InterfaceAudience; + import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.TimedQuota; -import org.apache.yetus.audience.InterfaceAudience; /** * Implementation of {@link GlobalQuotaSettings} to hide the Protobuf messages we use internally. @@ -45,18 +46,18 @@ public class GlobalQuotaSettingsImpl extends GlobalQuotaSettings { private final Boolean bypassGlobals; private final QuotaProtos.SpaceQuota spaceProto; - protected GlobalQuotaSettingsImpl( - String username, TableName tableName, String namespace, QuotaProtos.Quotas quotas) { - this(username, tableName, namespace, + protected GlobalQuotaSettingsImpl(String username, TableName tableName, String namespace, + String regionServer, QuotaProtos.Quotas quotas) { + this(username, tableName, namespace, regionServer, (quotas != null && quotas.hasThrottle() ? quotas.getThrottle() : null), (quotas != null && quotas.hasBypassGlobals() ? quotas.getBypassGlobals() : null), (quotas != null && quotas.hasSpace() ? quotas.getSpace() : null)); } - protected GlobalQuotaSettingsImpl( - String userName, TableName tableName, String namespace, QuotaProtos.Throttle throttleProto, - Boolean bypassGlobals, QuotaProtos.SpaceQuota spaceProto) { - super(userName, tableName, namespace); + protected GlobalQuotaSettingsImpl(String userName, TableName tableName, String namespace, + String regionServer, QuotaProtos.Throttle throttleProto, Boolean bypassGlobals, + QuotaProtos.SpaceQuota spaceProto) { + super(userName, tableName, namespace, regionServer); this.throttleProto = throttleProto; this.bypassGlobals = bypassGlobals; this.spaceProto = spaceProto; @@ -67,12 +68,12 @@ public class GlobalQuotaSettingsImpl extends GlobalQuotaSettings { // Very similar to QuotaSettingsFactory List settings = new ArrayList<>(); if (throttleProto != null) { - settings.addAll(QuotaSettingsFactory.fromThrottle( - getUserName(), getTableName(), getNamespace(), throttleProto)); + settings.addAll(QuotaSettingsFactory.fromThrottle(getUserName(), getTableName(), + getNamespace(), getRegionServer(), throttleProto)); } if (bypassGlobals != null && bypassGlobals.booleanValue()) { - settings.add(new QuotaGlobalsSettingsBypass( - getUserName(), getTableName(), getNamespace(), true)); + settings.add(new QuotaGlobalsSettingsBypass(getUserName(), getTableName(), getNamespace(), + getRegionServer(), true)); } if (spaceProto != null) { settings.add(QuotaSettingsFactory.fromSpace(getTableName(), getNamespace(), spaceProto)); @@ -210,7 +211,7 @@ public class GlobalQuotaSettingsImpl extends GlobalQuotaSettings { } return new GlobalQuotaSettingsImpl( - getUserName(), getTableName(), getNamespace(), + getUserName(), getTableName(), getNamespace(), getRegionServer(), (throttleBuilder == null ? null : throttleBuilder.build()), bypassGlobals, (removeSpaceBuilder ? null : spaceBuilder.build())); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java index bb3cff19918..389e1791dfb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java @@ -82,6 +82,7 @@ public class MasterQuotaManager implements RegionStateListener { private NamedLock namespaceLocks; private NamedLock tableLocks; private NamedLock userLocks; + private NamedLock regionServerLocks; private boolean initialized = false; private NamespaceAuditor namespaceQuotaManager; private ConcurrentHashMap regionSizes; @@ -110,6 +111,7 @@ public class MasterQuotaManager implements RegionStateListener { namespaceLocks = new NamedLock<>(); tableLocks = new NamedLock<>(); userLocks = new NamedLock<>(); + regionServerLocks = new NamedLock<>(); regionSizes = new ConcurrentHashMap<>(); namespaceQuotaManager = new NamespaceAuditor(masterServices); @@ -162,9 +164,16 @@ public class MasterQuotaManager implements RegionStateListener { } finally { namespaceLocks.unlock(req.getNamespace()); } + } else if (req.hasRegionServer()) { + regionServerLocks.lock(req.getRegionServer()); + try { + setRegionServerQuota(req.getRegionServer(), req); + } finally { + regionServerLocks.unlock(req.getRegionServer()); + } } else { - throw new DoNotRetryIOException( - new UnsupportedOperationException("a user, a table or a namespace must be specified")); + throw new DoNotRetryIOException(new UnsupportedOperationException( + "a user, a table, a namespace or region server must be specified")); } return SetQuotaResponse.newBuilder().build(); } @@ -174,8 +183,8 @@ public class MasterQuotaManager implements RegionStateListener { setQuota(req, new SetQuotaOperations() { @Override public GlobalQuotaSettingsImpl fetch() throws IOException { - return new GlobalQuotaSettingsImpl(req.getUserName(), null, null, QuotaUtil.getUserQuota( - masterServices.getConnection(), userName)); + return new GlobalQuotaSettingsImpl(req.getUserName(), null, null, null, + QuotaUtil.getUserQuota(masterServices.getConnection(), userName)); } @Override public void update(GlobalQuotaSettingsImpl quotaPojo) throws IOException { @@ -201,8 +210,8 @@ public class MasterQuotaManager implements RegionStateListener { setQuota(req, new SetQuotaOperations() { @Override public GlobalQuotaSettingsImpl fetch() throws IOException { - return new GlobalQuotaSettingsImpl(userName, table, null, QuotaUtil.getUserQuota( - masterServices.getConnection(), userName, table)); + return new GlobalQuotaSettingsImpl(userName, table, null, null, + QuotaUtil.getUserQuota(masterServices.getConnection(), userName, table)); } @Override public void update(GlobalQuotaSettingsImpl quotaPojo) throws IOException { @@ -229,8 +238,8 @@ public class MasterQuotaManager implements RegionStateListener { setQuota(req, new SetQuotaOperations() { @Override public GlobalQuotaSettingsImpl fetch() throws IOException { - return new GlobalQuotaSettingsImpl(userName, null, namespace, QuotaUtil.getUserQuota( - masterServices.getConnection(), userName, namespace)); + return new GlobalQuotaSettingsImpl(userName, null, namespace, null, + QuotaUtil.getUserQuota(masterServices.getConnection(), userName, namespace)); } @Override public void update(GlobalQuotaSettingsImpl quotaPojo) throws IOException { @@ -259,8 +268,8 @@ public class MasterQuotaManager implements RegionStateListener { setQuota(req, new SetQuotaOperations() { @Override public GlobalQuotaSettingsImpl fetch() throws IOException { - return new GlobalQuotaSettingsImpl(null, table, null, QuotaUtil.getTableQuota( - masterServices.getConnection(), table)); + return new GlobalQuotaSettingsImpl(null, table, null, null, + QuotaUtil.getTableQuota(masterServices.getConnection(), table)); } @Override public void update(GlobalQuotaSettingsImpl quotaPojo) throws IOException { @@ -286,8 +295,8 @@ public class MasterQuotaManager implements RegionStateListener { setQuota(req, new SetQuotaOperations() { @Override public GlobalQuotaSettingsImpl fetch() throws IOException { - return new GlobalQuotaSettingsImpl(null, null, namespace, QuotaUtil.getNamespaceQuota( - masterServices.getConnection(), namespace)); + return new GlobalQuotaSettingsImpl(null, null, namespace, null, + QuotaUtil.getNamespaceQuota(masterServices.getConnection(), namespace)); } @Override public void update(GlobalQuotaSettingsImpl quotaPojo) throws IOException { @@ -309,6 +318,38 @@ public class MasterQuotaManager implements RegionStateListener { }); } + public void setRegionServerQuota(final String regionServer, final SetQuotaRequest req) + throws IOException, InterruptedException { + setQuota(req, new SetQuotaOperations() { + @Override + public GlobalQuotaSettingsImpl fetch() throws IOException { + return new GlobalQuotaSettingsImpl(null, null, null, regionServer, + QuotaUtil.getRegionServerQuota(masterServices.getConnection(), regionServer)); + } + + @Override + public void update(GlobalQuotaSettingsImpl quotaPojo) throws IOException { + QuotaUtil.addRegionServerQuota(masterServices.getConnection(), regionServer, + ((GlobalQuotaSettingsImpl) quotaPojo).toQuotas()); + } + + @Override + public void delete() throws IOException { + QuotaUtil.deleteRegionServerQuota(masterServices.getConnection(), regionServer); + } + + @Override + public void preApply(GlobalQuotaSettingsImpl quotaPojo) throws IOException { + masterServices.getMasterCoprocessorHost().preSetRegionServerQuota(regionServer, quotaPojo); + } + + @Override + public void postApply(GlobalQuotaSettingsImpl quotaPojo) throws IOException { + masterServices.getMasterCoprocessorHost().postSetRegionServerQuota(regionServer, quotaPojo); + } + }); + } + public void setNamespaceQuota(NamespaceDescriptor desc) throws IOException { if (initialized) { this.namespaceQuotaManager.addNamespace(desc); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java index fac998f092d..4736362b1dd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java @@ -34,14 +34,14 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ScheduledChore; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableName; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.yetus.audience.InterfaceStability; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.yetus.audience.InterfaceAudience; +import org.apache.yetus.audience.InterfaceStability; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Cache that keeps track of the quota settings for the users and tables that @@ -69,6 +69,8 @@ public class QuotaCache implements Stoppable { private final ConcurrentHashMap namespaceQuotaCache = new ConcurrentHashMap<>(); private final ConcurrentHashMap tableQuotaCache = new ConcurrentHashMap<>(); private final ConcurrentHashMap userQuotaCache = new ConcurrentHashMap<>(); + private final ConcurrentHashMap regionServerQuotaCache = + new ConcurrentHashMap<>(); private final RegionServerServices rsServices; private QuotaRefresherChore refreshChore; @@ -146,6 +148,16 @@ public class QuotaCache implements Stoppable { return getQuotaState(this.namespaceQuotaCache, namespace).getGlobalLimiter(); } + /** + * Returns the limiter associated to the specified region server. + * + * @param regionServer the region server to limit + * @return the limiter associated to the specified region server + */ + public QuotaLimiter getRegionServerQuotaLimiter(final String regionServer) { + return getQuotaState(this.regionServerQuotaCache, regionServer).getGlobalLimiter(); + } + /** * Returns the QuotaState requested. If the quota info is not in cache an empty one will be * returned and the quota request will be enqueued for the next cache refresh. @@ -170,6 +182,11 @@ public class QuotaCache implements Stoppable { return namespaceQuotaCache; } + @VisibleForTesting + Map getRegionServerQuotaCache() { + return regionServerQuotaCache; + } + @VisibleForTesting Map getTableQuotaCache() { return tableQuotaCache; @@ -203,10 +220,13 @@ public class QuotaCache implements Stoppable { QuotaCache.this.namespaceQuotaCache.putIfAbsent(ns, new QuotaState()); } } + QuotaCache.this.regionServerQuotaCache.putIfAbsent(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY, + new QuotaState()); fetchNamespaceQuotaState(); fetchTableQuotaState(); fetchUserQuotaState(); + fetchRegionServerQuotaState(); lastUpdate = EnvironmentEdgeManager.currentTime(); } @@ -257,6 +277,21 @@ public class QuotaCache implements Stoppable { }); } + private void fetchRegionServerQuotaState() { + fetch("regionServer", QuotaCache.this.regionServerQuotaCache, + new Fetcher() { + @Override + public Get makeGet(final Map.Entry entry) { + return QuotaUtil.makeGetForRegionServerQuotas(entry.getKey()); + } + + @Override + public Map fetchEntries(final List gets) throws IOException { + return QuotaUtil.fetchRegionServerQuotas(rsServices.getConnection(), gets); + } + }); + } + private void fetch(final String type, final ConcurrentHashMap quotasMap, final Fetcher fetcher) { long now = EnvironmentEdgeManager.currentTime(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java index c183610cd9d..e562336db88 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java @@ -29,10 +29,6 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.yetus.audience.InterfaceStability; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; @@ -40,10 +36,15 @@ import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.yetus.audience.InterfaceAudience; +import org.apache.yetus.audience.InterfaceStability; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; /** * Helper class to interact with the quota table @@ -142,6 +143,16 @@ public class QuotaUtil extends QuotaTableUtil { getSettingsQualifierForUserNamespace(namespace)); } + public static void addRegionServerQuota(final Connection connection, final String regionServer, + final Quotas data) throws IOException { + addQuotas(connection, getRegionServerRowKey(regionServer), data); + } + + public static void deleteRegionServerQuota(final Connection connection, final String regionServer) + throws IOException { + deleteQuotas(connection, getRegionServerRowKey(regionServer)); + } + private static void addQuotas(final Connection connection, final byte[] rowKey, final Quotas data) throws IOException { addQuotas(connection, rowKey, QUOTA_QUALIFIER_SETTINGS, data); @@ -232,6 +243,17 @@ public class QuotaUtil extends QuotaTableUtil { }); } + public static Map fetchRegionServerQuotas(final Connection connection, + final List gets) throws IOException { + return fetchGlobalQuotas("regionServer", connection, gets, new KeyFromRow() { + @Override + public String getKeyFromRow(final byte[] row) { + assert isRegionServerRowKey(row); + return getRegionServerFromRowKey(row); + } + }); + } + public static Map fetchGlobalQuotas(final String type, final Connection connection, final List gets, final KeyFromRow kfr) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java index 9b3d48aef5e..99ff5162978 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java @@ -22,19 +22,19 @@ import java.io.IOException; import java.util.List; import java.util.Optional; -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.yetus.audience.InterfaceStability; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ipc.RpcScheduler; import org.apache.hadoop.hbase.ipc.RpcServer; -import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.yetus.audience.InterfaceAudience; +import org.apache.yetus.audience.InterfaceStability; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; /** * Region Server Quota Manager. @@ -135,14 +135,17 @@ public class RegionServerRpcQuotaManager { } else { QuotaLimiter nsLimiter = quotaCache.getNamespaceLimiter(table.getNamespaceAsString()); QuotaLimiter tableLimiter = quotaCache.getTableLimiter(table); - useNoop &= tableLimiter.isBypass() && nsLimiter.isBypass(); + QuotaLimiter rsLimiter = quotaCache + .getRegionServerQuotaLimiter(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY); + useNoop &= tableLimiter.isBypass() && nsLimiter.isBypass() && rsLimiter.isBypass(); if (LOG.isTraceEnabled()) { - LOG.trace("get quota for ugi=" + ugi + " table=" + table + " userLimiter=" + - userLimiter + " tableLimiter=" + tableLimiter + " nsLimiter=" + nsLimiter); + LOG.trace("get quota for ugi=" + ugi + " table=" + table + " userLimiter=" + userLimiter + + " tableLimiter=" + tableLimiter + " nsLimiter=" + nsLimiter + " rsLimiter=" + + rsLimiter); } if (!useNoop) { return new DefaultOperationQuota(this.rsServices.getConfiguration(), userLimiter, - tableLimiter, nsLimiter); + tableLimiter, nsLimiter, rsLimiter); } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java index bbf129dab33..f99531c463c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java @@ -2513,6 +2513,12 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor, requirePermission(ctx, "setNamespaceQuota", Action.ADMIN); } + @Override + public void preSetRegionServerQuota(ObserverContext ctx, + final String regionServer, GlobalQuotaSettings quotas) throws IOException { + requirePermission(ctx, "setRegionServerQuota", Action.ADMIN); + } + @Override public ReplicationEndpoint postCreateReplicationEndPoint( ObserverContext ctx, ReplicationEndpoint endpoint) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestGlobalQuotaSettingsImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestGlobalQuotaSettingsImpl.java index bd8a94a4b69..55938dc5f5d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestGlobalQuotaSettingsImpl.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestGlobalQuotaSettingsImpl.java @@ -58,9 +58,9 @@ public class TestGlobalQuotaSettingsImpl { QuotaProtos.ThrottleRequest writeThrottle = QuotaProtos.ThrottleRequest.newBuilder() .setTimedQuota(writeQuota).setType(QuotaProtos.ThrottleType.WRITE_NUMBER).build(); - GlobalQuotaSettingsImpl settings = new GlobalQuotaSettingsImpl("joe", null, null, quota); + GlobalQuotaSettingsImpl settings = new GlobalQuotaSettingsImpl("joe", null, null, null, quota); GlobalQuotaSettingsImpl merged = settings.merge( - new ThrottleSettings("joe", null, null, writeThrottle)); + new ThrottleSettings("joe", null, null, null, writeThrottle)); QuotaProtos.Throttle mergedThrottle = merged.getThrottleProto(); // Verify the request throttle is in place @@ -80,7 +80,7 @@ public class TestGlobalQuotaSettingsImpl { QuotaProtos.Quotas quota = QuotaProtos.Quotas.newBuilder() .setSpace(SPACE_QUOTA).build(); - GlobalQuotaSettingsImpl settings = new GlobalQuotaSettingsImpl(null, tn, null, quota); + GlobalQuotaSettingsImpl settings = new GlobalQuotaSettingsImpl(null, tn, null, null, quota); // Switch the violation policy to DISABLE GlobalQuotaSettingsImpl merged = settings.merge( new SpaceLimitSettings(tn, SPACE_QUOTA.getSoftLimit(), SpaceViolationPolicy.DISABLE)); @@ -96,7 +96,7 @@ public class TestGlobalQuotaSettingsImpl { final String ns = "org1"; QuotaProtos.Quotas quota = QuotaProtos.Quotas.newBuilder() .setThrottle(THROTTLE).setSpace(SPACE_QUOTA).build(); - GlobalQuotaSettingsImpl settings = new GlobalQuotaSettingsImpl(null, null, ns, quota); + GlobalQuotaSettingsImpl settings = new GlobalQuotaSettingsImpl(null, null, ns, null, quota); QuotaProtos.TimedQuota writeQuota = REQUEST_THROTTLE.toBuilder() .setSoftLimit(500).build(); @@ -105,7 +105,7 @@ public class TestGlobalQuotaSettingsImpl { .setTimedQuota(writeQuota).setType(QuotaProtos.ThrottleType.WRITE_NUMBER).build(); GlobalQuotaSettingsImpl merged = settings.merge( - new ThrottleSettings(null, null, ns, writeThrottle)); + new ThrottleSettings(null, null, ns, null, writeThrottle)); GlobalQuotaSettingsImpl finalQuota = merged.merge(new SpaceLimitSettings( ns, SPACE_QUOTA.getSoftLimit(), SpaceViolationPolicy.NO_WRITES_COMPACTIONS)); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java index ef7e47d66f8..88cd12719bb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java @@ -174,6 +174,7 @@ public class TestQuotaAdmin { assertEquals(userName, throttle.getUserName()); assertEquals(null, throttle.getTableName()); assertEquals(null, throttle.getNamespace()); + assertEquals(null, throttle.getRegionServer()); assertEquals(6, throttle.getSoftLimit()); assertEquals(TimeUnit.MINUTES, throttle.getTimeUnit()); countThrottle++; @@ -523,6 +524,51 @@ public class TestQuotaAdmin { } + @Test + public void testSetAndRemoveRegionServerQuota() throws Exception { + Admin admin = TEST_UTIL.getAdmin(); + String regionServer = QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY; + QuotaFilter rsFilter = new QuotaFilter().setRegionServerFilter(regionServer); + + admin.setQuota(QuotaSettingsFactory.throttleRegionServer(regionServer, + ThrottleType.REQUEST_NUMBER, 10, TimeUnit.MINUTES)); + assertNumResults(1, rsFilter); + // Verify the Quota in the table + verifyRecordPresentInQuotaTable(ThrottleType.REQUEST_NUMBER, 10, TimeUnit.MINUTES); + + admin.setQuota(QuotaSettingsFactory.throttleRegionServer(regionServer, + ThrottleType.REQUEST_NUMBER, 20, TimeUnit.MINUTES)); + assertNumResults(1, rsFilter); + // Verify the Quota in the table + verifyRecordPresentInQuotaTable(ThrottleType.REQUEST_NUMBER, 20, TimeUnit.MINUTES); + + admin.setQuota(QuotaSettingsFactory.throttleRegionServer(regionServer, ThrottleType.READ_NUMBER, + 30, TimeUnit.SECONDS)); + int count = 0; + QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration(), rsFilter); + try { + for (QuotaSettings settings : scanner) { + assertTrue(settings.getQuotaType() == QuotaType.THROTTLE); + ThrottleSettings throttleSettings = (ThrottleSettings) settings; + assertEquals(regionServer, throttleSettings.getRegionServer()); + count++; + if (throttleSettings.getThrottleType() == ThrottleType.REQUEST_NUMBER) { + assertEquals(20, throttleSettings.getSoftLimit()); + assertEquals(TimeUnit.MINUTES, throttleSettings.getTimeUnit()); + } else if (throttleSettings.getThrottleType() == ThrottleType.READ_NUMBER) { + assertEquals(30, throttleSettings.getSoftLimit()); + assertEquals(TimeUnit.SECONDS, throttleSettings.getTimeUnit()); + } + } + } finally { + scanner.close(); + } + assertEquals(2, count); + + admin.setQuota(QuotaSettingsFactory.unthrottleRegionServer(regionServer)); + assertNumResults(0, new QuotaFilter().setRegionServerFilter(regionServer)); + } + @Test public void testRpcThrottleWhenStartup() throws IOException, InterruptedException { TEST_UTIL.getAdmin().switchRpcThrottle(false); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java index c0694031489..aee97070d21 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaThrottle.java @@ -570,6 +570,31 @@ public class TestQuotaThrottle { triggerTableCacheRefresh(true, TABLE_NAMES[0]); } + @Test + public void testRegionServerThrottle() throws Exception { + final Admin admin = TEST_UTIL.getAdmin(); + admin.setQuota(QuotaSettingsFactory.throttleTable(TABLE_NAMES[0], ThrottleType.WRITE_NUMBER, 5, + TimeUnit.MINUTES)); + + // requests are throttled by table quota + admin.setQuota(QuotaSettingsFactory.throttleRegionServer( + QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY, ThrottleType.WRITE_NUMBER, 7, TimeUnit.MINUTES)); + triggerCacheRefresh(false, false, true, false, true, TABLE_NAMES[0]); + assertEquals(5, doPuts(10, tables[0])); + + // requests are throttled by region server quota + admin.setQuota(QuotaSettingsFactory.throttleRegionServer( + QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY, ThrottleType.WRITE_NUMBER, 4, TimeUnit.MINUTES)); + triggerCacheRefresh(false, false, false, false, true, TABLE_NAMES[0]); + assertEquals(4, doPuts(10, tables[0])); + + // unthrottle + admin.setQuota( + QuotaSettingsFactory.unthrottleRegionServer(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY)); + admin.setQuota(QuotaSettingsFactory.unthrottleTable(TABLE_NAMES[0])); + triggerCacheRefresh(true, false, true, false, true, TABLE_NAMES[0]); + } + private int doPuts(int maxOps, final Table... tables) throws Exception { return doPuts(maxOps, -1, tables); } @@ -622,19 +647,19 @@ public class TestQuotaThrottle { } private void triggerUserCacheRefresh(boolean bypass, TableName... tables) throws Exception { - triggerCacheRefresh(bypass, true, false, false, tables); + triggerCacheRefresh(bypass, true, false, false, false, tables); } private void triggerTableCacheRefresh(boolean bypass, TableName... tables) throws Exception { - triggerCacheRefresh(bypass, false, true, false, tables); + triggerCacheRefresh(bypass, false, true, false, false, tables); } private void triggerNamespaceCacheRefresh(boolean bypass, TableName... tables) throws Exception { - triggerCacheRefresh(bypass, false, false, true, tables); + triggerCacheRefresh(bypass, false, false, true, false, tables); } private void triggerCacheRefresh(boolean bypass, boolean userLimiter, boolean tableLimiter, - boolean nsLimiter, final TableName... tables) throws Exception { + boolean nsLimiter, boolean rsLimiter, final TableName... tables) throws Exception { envEdge.incValue(2 * REFRESH_TIME); for (RegionServerThread rst: TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) { RegionServerRpcQuotaManager quotaManager = rst.getRegionServer().getRegionServerRpcQuotaManager(); @@ -663,6 +688,11 @@ public class TestQuotaThrottle { if (nsLimiter) { isBypass &= quotaCache.getNamespaceLimiter(table.getNamespaceAsString()).isBypass(); } + if (rsLimiter) { + isBypass &= quotaCache + .getRegionServerQuotaLimiter(QuotaTableUtil.QUOTA_REGION_SERVER_ROW_KEY) + .isBypass(); + } if (isBypass != bypass) { envEdge.incValue(100); isUpdated = false; @@ -675,6 +705,7 @@ public class TestQuotaThrottle { LOG.debug(Objects.toString(quotaCache.getNamespaceQuotaCache())); LOG.debug(Objects.toString(quotaCache.getTableQuotaCache())); LOG.debug(Objects.toString(quotaCache.getUserQuotaCache())); + LOG.debug(Objects.toString(quotaCache.getRegionServerQuotaCache())); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java index 2d37f305aea..9b8c8a6fad7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java @@ -2700,6 +2700,15 @@ public class TestAccessController extends SecureTestUtil { } }; + AccessTestAction setRegionServerQuotaAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preSetRegionServerQuota(ObserverContextImpl.createAndPrepare(CP_ENV), + null, null); + return null; + } + }; + verifyAllowed(setUserQuotaAction, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); verifyDenied(setUserQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); @@ -2718,6 +2727,10 @@ public class TestAccessController extends SecureTestUtil { verifyAllowed(setNamespaceQuotaAction, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); verifyDenied(setNamespaceQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER, USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); + + verifyAllowed(setRegionServerQuotaAction, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); + verifyDenied(setRegionServerQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER, + USER_GROUP_READ, USER_GROUP_WRITE, USER_GROUP_CREATE); } @Test diff --git a/hbase-shell/src/main/ruby/hbase/quotas.rb b/hbase-shell/src/main/ruby/hbase/quotas.rb index 0534b7c0112..4023aedc1cb 100644 --- a/hbase-shell/src/main/ruby/hbase/quotas.rb +++ b/hbase-shell/src/main/ruby/hbase/quotas.rb @@ -82,8 +82,11 @@ module Hbase namespace = args.delete(NAMESPACE) raise(ArgumentError, 'Unexpected arguments: ' + args.inspect) unless args.empty? settings = QuotaSettingsFactory.throttleNamespace(namespace, type, limit, time_unit) + elsif args.key?(REGIONSERVER) + # TODO: Setting specified region server quota isn't supported currently and using 'all' for all RS + settings = QuotaSettingsFactory.throttleRegionServer('all', type, limit, time_unit) else - raise 'One of USER, TABLE or NAMESPACE must be specified' + raise 'One of USER, TABLE, NAMESPACE or REGIONSERVER must be specified' end @admin.setQuota(settings) end @@ -112,8 +115,13 @@ module Hbase namespace = args.delete(NAMESPACE) raise(ArgumentError, 'Unexpected arguments: ' + args.inspect) unless args.empty? settings = QuotaSettingsFactory.unthrottleNamespace(namespace) + elsif args.key?(REGIONSERVER) + regionServer = args.delete(REGIONSERVER) + raise(ArgumentError, 'Unexpected arguments: ' + args.inspect) unless args.empty? + # TODO: Setting specified region server quota isn't supported currently and using 'all' for all RS + settings = QuotaSettingsFactory.unthrottleRegionServer('all') else - raise 'One of USER, TABLE or NAMESPACE must be specified' + raise 'One of USER, TABLE, NAMESPACE or REGIONSERVER must be specified' end @admin.setQuota(settings) end @@ -233,7 +241,8 @@ module Hbase owner = { USER => settings.getUserName, TABLE => settings.getTableName, - NAMESPACE => settings.getNamespace + NAMESPACE => settings.getNamespace, + REGIONSERVER => settings.getRegionServer }.delete_if { |_k, v| v.nil? }.map { |k, v| k.to_s + ' => ' + v.to_s } * ', ' yield owner, settings.to_s diff --git a/hbase-shell/src/main/ruby/shell/commands/set_quota.rb b/hbase-shell/src/main/ruby/shell/commands/set_quota.rb index 3a5c136fa5b..696fa2a2a84 100644 --- a/hbase-shell/src/main/ruby/shell/commands/set_quota.rb +++ b/hbase-shell/src/main/ruby/shell/commands/set_quota.rb @@ -22,7 +22,7 @@ module Shell class SetQuota < Command def help <<-EOF -Set a quota for a user, table, or namespace. +Set a quota for a user, table, namespace or region server. Syntax : set_quota TYPE => , TYPE => THROTTLE @@ -55,6 +55,10 @@ For example: hbase> set_quota TYPE => THROTTLE, USER => 'u1', LIMIT => NONE hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => WRITE, USER => 'u1', LIMIT => NONE + hbase> set_quota TYPE => THROTTLE, REGIONSERVER => 'all', LIMIT => '30000req/sec' + hbase> set_quota TYPE => THROTTLE, REGIONSERVER => 'all', THROTTLE_TYPE => WRITE, LIMIT => '20000req/sec' + hbase> set_quota TYPE => THROTTLE, REGIONSERVER => 'all', LIMIT => NONE + hbase> set_quota USER => 'u1', GLOBAL_BYPASS => true TYPE => SPACE diff --git a/hbase-shell/src/test/ruby/hbase/quotas_test.rb b/hbase-shell/src/test/ruby/hbase/quotas_test.rb index 1dd215dee51..1bcc07b7e58 100644 --- a/hbase-shell/src/test/ruby/hbase/quotas_test.rb +++ b/hbase-shell/src/test/ruby/hbase/quotas_test.rb @@ -251,6 +251,27 @@ module Hbase output = capture_stdout { command(:enable_rpc_throttle) } assert(output.include?('Previous rpc throttle state : false')) end + + define_test 'can set and remove region server quota' do + command(:set_quota, TYPE => THROTTLE, REGIONSERVER => 'all', LIMIT => '1CU/sec') + output = capture_stdout{ command(:list_quotas) } + assert(output.include?('REGIONSERVER => all')) + assert(output.include?('TYPE => THROTTLE')) + assert(output.include?('THROTTLE_TYPE => REQUEST_CAPACITY_UNIT')) + assert(output.include?('LIMIT => 1CU/sec')) + + command(:set_quota, TYPE => THROTTLE, REGIONSERVER => 'all', THROTTLE_TYPE => WRITE, LIMIT => '2req/sec') + output = capture_stdout{ command(:list_quotas) } + assert(output.include?('REGIONSERVER => all')) + assert(output.include?('TYPE => THROTTLE')) + assert(output.include?('THROTTLE_TYPE => WRITE')) + assert(output.include?('THROTTLE_TYPE => REQUEST_CAPACITY_UNIT')) + assert(output.include?('LIMIT => 2req/sec')) + + command(:set_quota, TYPE => THROTTLE, REGIONSERVER => 'all', LIMIT => NONE) + output = capture_stdout{ command(:list_quotas) } + assert(output.include?('0 row(s)')) + end end # rubocop:enable Metrics/ClassLength end