Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-17025 Add shell commands for space quotas

This commit is contained in:
Josh Elser 2017-01-11 11:55:29 -05:00
parent 6c9082fe16
commit f1066cd774
6 changed files with 242 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
hbase-shell/src/main/ruby/hbase/quotas.rb
View File

@ -24,14 +24,22 @@ java_import org.apache.hadoop.hbase.quotas.ThrottleType
java_import org.apache.hadoop.hbase.quotas.QuotaFilter
java_import org.apache.hadoop.hbase.quotas.QuotaRetriever
java_import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory
java_import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy
module HBaseQuotasConstants
# RPC Quota constants
GLOBAL_BYPASS = 'GLOBAL_BYPASS'
THROTTLE_TYPE = 'THROTTLE_TYPE'
THROTTLE = 'THROTTLE'
REQUEST = 'REQUEST'
WRITE = 'WRITE'
READ = 'READ'
# Space quota constants
SPACE = 'SPACE'
NO_INSERTS = 'NO_INSERTS'
NO_WRITES = 'NO_WRITES'
NO_WRITES_COMPACTIONS = 'NO_WRITES_COMPACTIONS'
DISABLE = 'DISABLE'
end
module Hbase
@ -107,6 +115,54 @@ module Hbase
@admin.setQuota(settings)
end
def limit_space(args)
raise(ArgumentError, 'Argument should be a Hash') unless (not args.nil? and args.kind_of?(Hash))
# Let the user provide a raw number
if args[LIMIT].is_a?(Numeric)
limit = args[LIMIT]
else
# Parse a string a 1K, 2G, etc.
limit = _parse_size(args[LIMIT])
end
# Extract the policy, failing if something bogus was provided
policy = SpaceViolationPolicy.valueOf(args[POLICY])
# Create a table or namespace quota
if args.key?(TABLE)
if args.key?(NAMESPACE)
raise(ArgumentError, "Only one of TABLE or NAMESPACE can be specified.")
end
settings = QuotaSettingsFactory.limitTableSpace(TableName.valueOf(args.delete(TABLE)), limit, policy)
elsif args.key?(NAMESPACE)
if args.key?(TABLE)
raise(ArgumentError, "Only one of TABLE or NAMESPACE can be specified.")
end
settings = QuotaSettingsFactory.limitNamespaceSpace(args.delete(NAMESPACE), limit, policy)
else
raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.')
end
# Apply the quota
@admin.setQuota(settings)
end
def remove_space_limit(args)
raise(ArgumentError, 'Argument should be a Hash') unless (not args.nil? and args.kind_of?(Hash))
if args.key?(TABLE)
if args.key?(NAMESPACE)
raise(ArgumentError, "Only one of TABLE or NAMESPACE can be specified.")
end
table = TableName.valueOf(args.delete(TABLE))
settings = QuotaSettingsFactory.removeTableSpaceLimit(table)
elsif args.key?(NAMESPACE)
if args.key?(TABLE)
raise(ArgumentError, "Only one of TABLE or NAMESPACE can be specified.")
end
settings = QuotaSettingsFactory.removeNamespaceSpaceLimit(args.delete(NAMESPACE))
else
raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.')
end
@admin.setQuota(settings)
end
def set_global_bypass(bypass, args)
raise(ArgumentError, "Arguments should be a Hash") unless args.kind_of?(Hash)
@ -171,7 +227,7 @@ module Hbase
return _size_from_str(match[1].to_i, match[2])
end
else
raise "Invalid size limit syntax"
raise(ArgumentError, "Invalid size limit syntax")
end
end
@ -188,7 +244,7 @@ module Hbase
end
if limit <= 0
raise "Invalid throttle limit, must be greater then 0"
raise(ArgumentError, "Invalid throttle limit, must be greater then 0")
end
case match[3]
@ -200,7 +256,7 @@ module Hbase
return type, limit, time_unit
else
raise "Invalid throttle limit syntax"
raise(ArgumentError, "Invalid throttle limit syntax")
end
end

1
hbase-shell/src/main/ruby/hbase_constants.rb
View File

@ -86,6 +86,7 @@ module HBaseConstants
RESTORE_ACL = 'RESTORE_ACL'
FORMATTER = 'FORMATTER'
FORMATTER_CLASS = 'FORMATTER_CLASS'
POLICY = 'POLICY'
# Load constants from hbase java API
def self.promote_constants(constants)

45
hbase-shell/src/main/ruby/shell/commands/set_quota.rb
View File

@ -52,6 +52,37 @@ For example:
hbase> set_quota TYPE => THROTTLE, THROTTLE_TYPE => WRITE, USER => 'u1', LIMIT => NONE
hbase> set_quota USER => 'u1', GLOBAL_BYPASS => true
TYPE => SPACE
Users can either set a quota on a table or a namespace. The quota is a limit on the target's
size on the FileSystem and some action to take when the target exceeds that limit. The limit
is in bytes and can expressed using standard metric suffixes (B, K, M, G, T, P), defaulting
to bytes if not provided. Different quotas can be applied to one table at the table and namespace
level; table-level quotas take priority over namespace-level quotas.
There are a limited number of policies to take when a quota is violation, listed in order of
least strict to most strict.
NO_INSERTS - No new data is allowed to be ingested (e.g. Put, Increment, Append).
NO_WRITES - Same as NO_INSERTS but Deletes are also disallowed.
NO_WRITES_COMPACTIONS - Same as NO_WRITES but compactions are also disallowed.
DISABLE - The table(s) are disabled.
For example:
hbase> set_quota TYPE => SPACE, TABLE => 't1', LIMIT => '1G', POLICY => NO_INSERTS
hbase> set_quota TYPE => SPACE, TABLE => 't2', LIMIT => '50G', POLICY => DISABLE
hbase> set_quota TYPE => SPACE, TABLE => 't3', LIMIT => '2T', POLICY => NO_WRITES_COMPACTIONS
hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => '50T', POLICY => NO_WRITES
Space quotas can also be removed via this command. To remove a space quota, provide NONE
for the limit.
For example:
hbase> set_quota TYPE => SPACE, TABLE => 't1', LIMIT => NONE
hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => NONE
EOF
end
@ -66,8 +97,18 @@ EOF
else
quotas_admin.throttle(args)
end
else
raise "Invalid TYPE argument. got " + qtype
when SPACE
if args[LIMIT].eql? NONE
args.delete(LIMIT)
# Table/Namespace argument is verified in remove_space_limit
quotas_admin.remove_space_limit(args)
else
raise(ArgumentError, 'Expected a LIMIT to be provided') unless args.key?(LIMIT)
raise(ArgumentError, 'Expected a POLICY to be provided') unless args.key?(POLICY)
quotas_admin.limit_space(args)
end
else
raise "Invalid TYPE argument. got " + qtype
end
elsif args.has_key?(GLOBAL_BYPASS)
quotas_admin.set_global_bypass(args.delete(GLOBAL_BYPASS), args)

1
hbase-shell/src/test/java/org/apache/hadoop/hbase/client/AbstractTestShell.java
View File

@ -39,6 +39,7 @@ public abstract class AbstractTestShell {
// Start mini cluster
TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
TEST_UTIL.getConfiguration().setBoolean("hbase.quota.enabled", true);
TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
TEST_UTIL.getConfiguration().setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, false);
TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);

137
hbase-shell/src/test/ruby/hbase/quotas_test.rb Normal file
View File

@ -0,0 +1,137 @@
#
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
require 'shell'
require 'stringio'
require 'hbase_constants'
require 'hbase/hbase'
require 'hbase/table'
include HBaseConstants
module Hbase
class SpaceQuotasTest < Test::Unit::TestCase
include TestHelpers
def setup
setup_hbase
# Create test table if it does not exist
@test_name = "hbase_shell_tests_table"
create_test_table(@test_name)
end
def teardown
shutdown
end
define_test 'limit_space errors on non Hash argument' do
qa = quotas_admin()
assert_raise(ArgumentError) do
qa.limit_space('foo')
end
assert_raise(ArgumentError) do
qa.limit_space()
end
end
define_test 'remove_space_limit errors on non Hash argument' do
qa = quotas_admin()
assert_raise(ArgumentError) do
qa.remove_space_limit('foo')
end
assert_raise(ArgumentError) do
qa.remove_space_limit()
end
end
define_test 'set quota with a non-numeric limit fails' do
assert_raise(ArgumentError) do
command(:set_quota, TYPE => SPACE, LIMIT => 'asdf', POLICY => NO_INSERTS, TABLE => @test_name)
end
end
define_test 'set quota without a limit fails' do
assert_raise(ArgumentError) do
command(:set_quota, TYPE => SPACE, POLICY => NO_INSERTS, TABLE => @test_name)
end
end
define_test 'set quota without a policy fails' do
assert_raise(ArgumentError) do
command(:set_quota, TYPE => SPACE, LIMIT => '1G', TABLE => @test_name)
end
end
define_test 'set quota without a table or namespace fails' do
assert_raise(ArgumentError) do
command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => NO_INSERTS)
end
end
define_test 'invalid violation policy specified' do
assert_raise(NameError) do
command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => FOO_BAR, TABLE => @test_name)
end
end
define_test 'table and namespace are mutually exclusive in set quota' do
assert_raise(ArgumentError) do
command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => NO_INSERTS, TABLE => @test_name, NAMESPACE => "foo")
end
end
define_test '_parse_size accepts various forms of byte shorthand' do
qa = quotas_admin()
KILO = 1024
MEGA = KILO * KILO
GIGA = MEGA * KILO
TERA = GIGA * KILO
PETA = TERA * KILO
assert_equal(1, qa._parse_size("1"))
assert_equal(1, qa._parse_size("1b"))
assert_equal(1, qa._parse_size("1B"))
assert_equal(KILO * 2, qa._parse_size("2k"))
assert_equal(KILO * 2, qa._parse_size("2K"))
assert_equal(MEGA * 5, qa._parse_size("5m"))
assert_equal(MEGA * 5, qa._parse_size("5M"))
assert_equal(GIGA * 3, qa._parse_size("3g"))
assert_equal(GIGA * 3, qa._parse_size("3G"))
assert_equal(TERA * 4, qa._parse_size("4t"))
assert_equal(TERA * 4, qa._parse_size("4T"))
assert_equal(PETA * 32, qa._parse_size("32p"))
assert_equal(PETA * 32, qa._parse_size("32P"))
assert_equal(GIGA * 4, qa._parse_size("4096m"))
assert_equal(GIGA * 4, qa._parse_size("4096M"))
end
define_test 'can set and remove quota' do
command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => NO_INSERTS, TABLE => @test_name)
output = capture_stdout{ command(:list_quotas) }
size = 1024 * 1024 * 1024
assert(output.include?("LIMIT => #{size}"))
assert(output.include?("VIOLATION_POLICY => NO_INSERTS"))
assert(output.include?("TYPE => SPACE"))
assert(output.include?("TABLE => #{@test_name}"))
command(:set_quota, TYPE => SPACE, LIMIT => NONE, TABLE => @test_name)
output = capture_stdout{ command(:list_quotas) }
assert(output.include?("0 row(s)"))
end
end
end

1
hbase-shell/src/test/ruby/tests_runner.rb
View File

@ -36,6 +36,7 @@ unless defined?($TEST_CLUSTER)
$TEST_CLUSTER = HBaseTestingUtility.new
$TEST_CLUSTER.configuration.setInt("hbase.regionserver.msginterval", 100)
$TEST_CLUSTER.configuration.setInt("hbase.client.pause", 250)
$TEST_CLUSTER.configuration.set("hbase.quota.enabled", "true")
$TEST_CLUSTER.configuration.setInt(org.apache.hadoop.hbase.HConstants::HBASE_CLIENT_RETRIES_NUMBER, 6)
$TEST_CLUSTER.startMiniCluster
@own_cluster = true