From f1066cd7744e185044dcc33aae09623d10ed2631 Mon Sep 17 00:00:00 2001 From: Josh Elser Date: Wed, 11 Jan 2017 11:55:29 -0500 Subject: [PATCH] HBASE-17025 Add shell commands for space quotas --- hbase-shell/src/main/ruby/hbase/quotas.rb | 62 +++++++- hbase-shell/src/main/ruby/hbase_constants.rb | 1 + .../src/main/ruby/shell/commands/set_quota.rb | 45 +++++- .../hbase/client/AbstractTestShell.java | 1 + .../src/test/ruby/hbase/quotas_test.rb | 137 ++++++++++++++++++ hbase-shell/src/test/ruby/tests_runner.rb | 1 + 6 files changed, 242 insertions(+), 5 deletions(-) create mode 100644 hbase-shell/src/test/ruby/hbase/quotas_test.rb diff --git a/hbase-shell/src/main/ruby/hbase/quotas.rb b/hbase-shell/src/main/ruby/hbase/quotas.rb index bf2dc63d96a..d99fe72ed88 100644 --- a/hbase-shell/src/main/ruby/hbase/quotas.rb +++ b/hbase-shell/src/main/ruby/hbase/quotas.rb @@ -24,14 +24,22 @@ java_import org.apache.hadoop.hbase.quotas.ThrottleType java_import org.apache.hadoop.hbase.quotas.QuotaFilter java_import org.apache.hadoop.hbase.quotas.QuotaRetriever java_import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory +java_import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy module HBaseQuotasConstants + # RPC Quota constants GLOBAL_BYPASS = 'GLOBAL_BYPASS' THROTTLE_TYPE = 'THROTTLE_TYPE' THROTTLE = 'THROTTLE' REQUEST = 'REQUEST' WRITE = 'WRITE' READ = 'READ' + # Space quota constants + SPACE = 'SPACE' + NO_INSERTS = 'NO_INSERTS' + NO_WRITES = 'NO_WRITES' + NO_WRITES_COMPACTIONS = 'NO_WRITES_COMPACTIONS' + DISABLE = 'DISABLE' end module Hbase @@ -107,6 +115,54 @@ module Hbase @admin.setQuota(settings) end + def limit_space(args) + raise(ArgumentError, 'Argument should be a Hash') unless (not args.nil? and args.kind_of?(Hash)) + # Let the user provide a raw number + if args[LIMIT].is_a?(Numeric) + limit = args[LIMIT] + else + # Parse a string a 1K, 2G, etc. + limit = _parse_size(args[LIMIT]) + end + # Extract the policy, failing if something bogus was provided + policy = SpaceViolationPolicy.valueOf(args[POLICY]) + # Create a table or namespace quota + if args.key?(TABLE) + if args.key?(NAMESPACE) + raise(ArgumentError, "Only one of TABLE or NAMESPACE can be specified.") + end + settings = QuotaSettingsFactory.limitTableSpace(TableName.valueOf(args.delete(TABLE)), limit, policy) + elsif args.key?(NAMESPACE) + if args.key?(TABLE) + raise(ArgumentError, "Only one of TABLE or NAMESPACE can be specified.") + end + settings = QuotaSettingsFactory.limitNamespaceSpace(args.delete(NAMESPACE), limit, policy) + else + raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.') + end + # Apply the quota + @admin.setQuota(settings) + end + + def remove_space_limit(args) + raise(ArgumentError, 'Argument should be a Hash') unless (not args.nil? and args.kind_of?(Hash)) + if args.key?(TABLE) + if args.key?(NAMESPACE) + raise(ArgumentError, "Only one of TABLE or NAMESPACE can be specified.") + end + table = TableName.valueOf(args.delete(TABLE)) + settings = QuotaSettingsFactory.removeTableSpaceLimit(table) + elsif args.key?(NAMESPACE) + if args.key?(TABLE) + raise(ArgumentError, "Only one of TABLE or NAMESPACE can be specified.") + end + settings = QuotaSettingsFactory.removeNamespaceSpaceLimit(args.delete(NAMESPACE)) + else + raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.') + end + @admin.setQuota(settings) + end + def set_global_bypass(bypass, args) raise(ArgumentError, "Arguments should be a Hash") unless args.kind_of?(Hash) @@ -171,7 +227,7 @@ module Hbase return _size_from_str(match[1].to_i, match[2]) end else - raise "Invalid size limit syntax" + raise(ArgumentError, "Invalid size limit syntax") end end @@ -188,7 +244,7 @@ module Hbase end if limit <= 0 - raise "Invalid throttle limit, must be greater then 0" + raise(ArgumentError, "Invalid throttle limit, must be greater then 0") end case match[3] @@ -200,7 +256,7 @@ module Hbase return type, limit, time_unit else - raise "Invalid throttle limit syntax" + raise(ArgumentError, "Invalid throttle limit syntax") end end diff --git a/hbase-shell/src/main/ruby/hbase_constants.rb b/hbase-shell/src/main/ruby/hbase_constants.rb index 7d6da9f06ba..fe8e812932e 100644 --- a/hbase-shell/src/main/ruby/hbase_constants.rb +++ b/hbase-shell/src/main/ruby/hbase_constants.rb @@ -86,6 +86,7 @@ module HBaseConstants RESTORE_ACL = 'RESTORE_ACL' FORMATTER = 'FORMATTER' FORMATTER_CLASS = 'FORMATTER_CLASS' + POLICY = 'POLICY' # Load constants from hbase java API def self.promote_constants(constants) diff --git a/hbase-shell/src/main/ruby/shell/commands/set_quota.rb b/hbase-shell/src/main/ruby/shell/commands/set_quota.rb index a638b939702..06ed0baff40 100644 --- a/hbase-shell/src/main/ruby/shell/commands/set_quota.rb +++ b/hbase-shell/src/main/ruby/shell/commands/set_quota.rb @@ -52,6 +52,37 @@ For example: hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => WRITE, USER => 'u1', LIMIT => NONE hbase> set_quota USER => 'u1', GLOBAL_BYPASS => true + +TYPE => SPACE +Users can either set a quota on a table or a namespace. The quota is a limit on the target's +size on the FileSystem and some action to take when the target exceeds that limit. The limit +is in bytes and can expressed using standard metric suffixes (B, K, M, G, T, P), defaulting +to bytes if not provided. Different quotas can be applied to one table at the table and namespace +level; table-level quotas take priority over namespace-level quotas. + +There are a limited number of policies to take when a quota is violation, listed in order of +least strict to most strict. + + NO_INSERTS - No new data is allowed to be ingested (e.g. Put, Increment, Append). + NO_WRITES - Same as NO_INSERTS but Deletes are also disallowed. + NO_WRITES_COMPACTIONS - Same as NO_WRITES but compactions are also disallowed. + DISABLE - The table(s) are disabled. + +For example: + + hbase> set_quota TYPE => SPACE, TABLE => 't1', LIMIT => '1G', POLICY => NO_INSERTS + hbase> set_quota TYPE => SPACE, TABLE => 't2', LIMIT => '50G', POLICY => DISABLE + hbase> set_quota TYPE => SPACE, TABLE => 't3', LIMIT => '2T', POLICY => NO_WRITES_COMPACTIONS + hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => '50T', POLICY => NO_WRITES + +Space quotas can also be removed via this command. To remove a space quota, provide NONE +for the limit. + +For example: + + hbase> set_quota TYPE => SPACE, TABLE => 't1', LIMIT => NONE + hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => NONE + EOF end @@ -66,8 +97,18 @@ EOF else quotas_admin.throttle(args) end - else - raise "Invalid TYPE argument. got " + qtype + when SPACE + if args[LIMIT].eql? NONE + args.delete(LIMIT) + # Table/Namespace argument is verified in remove_space_limit + quotas_admin.remove_space_limit(args) + else + raise(ArgumentError, 'Expected a LIMIT to be provided') unless args.key?(LIMIT) + raise(ArgumentError, 'Expected a POLICY to be provided') unless args.key?(POLICY) + quotas_admin.limit_space(args) + end + else + raise "Invalid TYPE argument. got " + qtype end elsif args.has_key?(GLOBAL_BYPASS) quotas_admin.set_global_bypass(args.delete(GLOBAL_BYPASS), args) diff --git a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java index 140380502cd..53606e97d5a 100644 --- a/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java +++ b/hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java @@ -39,6 +39,7 @@ public abstract class AbstractTestShell { // Start mini cluster TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); + TEST_UTIL.getConfiguration().setBoolean("hbase.quota.enabled", true); TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); TEST_UTIL.getConfiguration().setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, false); TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3); diff --git a/hbase-shell/src/test/ruby/hbase/quotas_test.rb b/hbase-shell/src/test/ruby/hbase/quotas_test.rb new file mode 100644 index 00000000000..78c889c67af --- /dev/null +++ b/hbase-shell/src/test/ruby/hbase/quotas_test.rb @@ -0,0 +1,137 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'shell' +require 'stringio' +require 'hbase_constants' +require 'hbase/hbase' +require 'hbase/table' + +include HBaseConstants + +module Hbase + class SpaceQuotasTest < Test::Unit::TestCase + include TestHelpers + + def setup + setup_hbase + # Create test table if it does not exist + @test_name = "hbase_shell_tests_table" + create_test_table(@test_name) + end + + def teardown + shutdown + end + + define_test 'limit_space errors on non Hash argument' do + qa = quotas_admin() + assert_raise(ArgumentError) do + qa.limit_space('foo') + end + assert_raise(ArgumentError) do + qa.limit_space() + end + end + + define_test 'remove_space_limit errors on non Hash argument' do + qa = quotas_admin() + assert_raise(ArgumentError) do + qa.remove_space_limit('foo') + end + assert_raise(ArgumentError) do + qa.remove_space_limit() + end + end + + define_test 'set quota with a non-numeric limit fails' do + assert_raise(ArgumentError) do + command(:set_quota, TYPE => SPACE, LIMIT => 'asdf', POLICY => NO_INSERTS, TABLE => @test_name) + end + end + + define_test 'set quota without a limit fails' do + assert_raise(ArgumentError) do + command(:set_quota, TYPE => SPACE, POLICY => NO_INSERTS, TABLE => @test_name) + end + end + + define_test 'set quota without a policy fails' do + assert_raise(ArgumentError) do + command(:set_quota, TYPE => SPACE, LIMIT => '1G', TABLE => @test_name) + end + end + + define_test 'set quota without a table or namespace fails' do + assert_raise(ArgumentError) do + command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => NO_INSERTS) + end + end + + define_test 'invalid violation policy specified' do + assert_raise(NameError) do + command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => FOO_BAR, TABLE => @test_name) + end + end + + define_test 'table and namespace are mutually exclusive in set quota' do + assert_raise(ArgumentError) do + command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => NO_INSERTS, TABLE => @test_name, NAMESPACE => "foo") + end + end + + define_test '_parse_size accepts various forms of byte shorthand' do + qa = quotas_admin() + KILO = 1024 + MEGA = KILO * KILO + GIGA = MEGA * KILO + TERA = GIGA * KILO + PETA = TERA * KILO + assert_equal(1, qa._parse_size("1")) + assert_equal(1, qa._parse_size("1b")) + assert_equal(1, qa._parse_size("1B")) + assert_equal(KILO * 2, qa._parse_size("2k")) + assert_equal(KILO * 2, qa._parse_size("2K")) + assert_equal(MEGA * 5, qa._parse_size("5m")) + assert_equal(MEGA * 5, qa._parse_size("5M")) + assert_equal(GIGA * 3, qa._parse_size("3g")) + assert_equal(GIGA * 3, qa._parse_size("3G")) + assert_equal(TERA * 4, qa._parse_size("4t")) + assert_equal(TERA * 4, qa._parse_size("4T")) + assert_equal(PETA * 32, qa._parse_size("32p")) + assert_equal(PETA * 32, qa._parse_size("32P")) + assert_equal(GIGA * 4, qa._parse_size("4096m")) + assert_equal(GIGA * 4, qa._parse_size("4096M")) + end + + define_test 'can set and remove quota' do + command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => NO_INSERTS, TABLE => @test_name) + output = capture_stdout{ command(:list_quotas) } + size = 1024 * 1024 * 1024 + assert(output.include?("LIMIT => #{size}")) + assert(output.include?("VIOLATION_POLICY => NO_INSERTS")) + assert(output.include?("TYPE => SPACE")) + assert(output.include?("TABLE => #{@test_name}")) + + command(:set_quota, TYPE => SPACE, LIMIT => NONE, TABLE => @test_name) + output = capture_stdout{ command(:list_quotas) } + assert(output.include?("0 row(s)")) + end + end +end diff --git a/hbase-shell/src/test/ruby/tests_runner.rb b/hbase-shell/src/test/ruby/tests_runner.rb index 74ddb485131..54bf3f9796d 100644 --- a/hbase-shell/src/test/ruby/tests_runner.rb +++ b/hbase-shell/src/test/ruby/tests_runner.rb @@ -36,6 +36,7 @@ unless defined?($TEST_CLUSTER) $TEST_CLUSTER = HBaseTestingUtility.new $TEST_CLUSTER.configuration.setInt("hbase.regionserver.msginterval", 100) $TEST_CLUSTER.configuration.setInt("hbase.client.pause", 250) + $TEST_CLUSTER.configuration.set("hbase.quota.enabled", "true") $TEST_CLUSTER.configuration.setInt(org.apache.hadoop.hbase.HConstants::HBASE_CLIENT_RETRIES_NUMBER, 6) $TEST_CLUSTER.startMiniCluster @own_cluster = true