HBASE-21549 Add shell command for serial replication peer
This commit is contained in:
parent
dfb9ae8e0e
commit
1e65bd5cf9
|
@ -66,6 +66,7 @@ module Hbase
|
||||||
namespaces = args.fetch(NAMESPACES, nil)
|
namespaces = args.fetch(NAMESPACES, nil)
|
||||||
peer_state = args.fetch(STATE, nil)
|
peer_state = args.fetch(STATE, nil)
|
||||||
remote_wal_dir = args.fetch(REMOTE_WAL_DIR, nil)
|
remote_wal_dir = args.fetch(REMOTE_WAL_DIR, nil)
|
||||||
|
serial = args.fetch(SERIAL, nil)
|
||||||
|
|
||||||
# Create and populate a ReplicationPeerConfig
|
# Create and populate a ReplicationPeerConfig
|
||||||
builder = ReplicationPeerConfig.newBuilder()
|
builder = ReplicationPeerConfig.newBuilder()
|
||||||
|
@ -79,6 +80,10 @@ module Hbase
|
||||||
builder.setRemoteWALDir(remote_wal_dir)
|
builder.setRemoteWALDir(remote_wal_dir)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
unless serial.nil?
|
||||||
|
builder.setSerial(serial)
|
||||||
|
end
|
||||||
|
|
||||||
unless config.nil?
|
unless config.nil?
|
||||||
builder.putAllConfiguration(config)
|
builder.putAllConfiguration(config)
|
||||||
end
|
end
|
||||||
|
|
|
@ -78,6 +78,7 @@ module HBaseConstants
|
||||||
ENDPOINT_CLASSNAME = 'ENDPOINT_CLASSNAME'.freeze
|
ENDPOINT_CLASSNAME = 'ENDPOINT_CLASSNAME'.freeze
|
||||||
CLUSTER_KEY = 'CLUSTER_KEY'.freeze
|
CLUSTER_KEY = 'CLUSTER_KEY'.freeze
|
||||||
REMOTE_WAL_DIR = 'REMOTE_WAL_DIR'.freeze
|
REMOTE_WAL_DIR = 'REMOTE_WAL_DIR'.freeze
|
||||||
|
SERIAL = 'SERIAL'.freeze
|
||||||
TABLE_CFS = 'TABLE_CFS'.freeze
|
TABLE_CFS = 'TABLE_CFS'.freeze
|
||||||
NAMESPACES = 'NAMESPACES'.freeze
|
NAMESPACES = 'NAMESPACES'.freeze
|
||||||
STATE = 'STATE'.freeze
|
STATE = 'STATE'.freeze
|
||||||
|
|
|
@ -34,6 +34,8 @@ An optional parameter for namespaces identifies which namespace's tables will be
|
||||||
to the peer cluster.
|
to the peer cluster.
|
||||||
An optional parameter for table column families identifies which tables and/or column families
|
An optional parameter for table column families identifies which tables and/or column families
|
||||||
will be replicated to the peer cluster.
|
will be replicated to the peer cluster.
|
||||||
|
An optional parameter for serial flag identifies whether or not the replication peer is a serial
|
||||||
|
replication peer. The default serial flag is false.
|
||||||
|
|
||||||
Note: Set a namespace in the peer config means that all tables in this namespace
|
Note: Set a namespace in the peer config means that all tables in this namespace
|
||||||
will be replicated to the peer cluster. So if you already have set a namespace in peer config,
|
will be replicated to the peer cluster. So if you already have set a namespace in peer config,
|
||||||
|
@ -50,6 +52,8 @@ Examples:
|
||||||
NAMESPACES => ["ns1", "ns2", "ns3"]
|
NAMESPACES => ["ns1", "ns2", "ns3"]
|
||||||
hbase> add_peer '2', CLUSTER_KEY => "zk1,zk2,zk3:2182:/hbase-prod",
|
hbase> add_peer '2', CLUSTER_KEY => "zk1,zk2,zk3:2182:/hbase-prod",
|
||||||
NAMESPACES => ["ns1", "ns2"], TABLE_CFS => { "ns3:table1" => [], "ns3:table2" => ["cf1"] }
|
NAMESPACES => ["ns1", "ns2"], TABLE_CFS => { "ns3:table1" => [], "ns3:table2" => ["cf1"] }
|
||||||
|
hbase> add_peer '3', CLUSTER_KEY => "zk1,zk2,zk3:2182:/hbase-prod",
|
||||||
|
NAMESPACES => ["ns1", "ns2", "ns3"], SERIAL => true
|
||||||
|
|
||||||
For a custom replication endpoint, the ENDPOINT_CLASSNAME can be provided. Two optional arguments
|
For a custom replication endpoint, the ENDPOINT_CLASSNAME can be provided. Two optional arguments
|
||||||
are DATA and CONFIG which can be specified to set different either the peer_data or configuration
|
are DATA and CONFIG which can be specified to set different either the peer_data or configuration
|
||||||
|
|
|
@ -41,8 +41,8 @@ module Shell
|
||||||
EOF
|
EOF
|
||||||
end
|
end
|
||||||
|
|
||||||
def command(id, peer_serial)
|
def command(id, serial)
|
||||||
replication_admin.set_peer_serial(id, peer_serial)
|
replication_admin.set_peer_serial(id, serial)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -100,6 +100,27 @@ module Hbase
|
||||||
command(:remove_peer, @peer_id)
|
command(:remove_peer, @peer_id)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
define_test "add_peer: serial" do
|
||||||
|
cluster_key = "server1.cie.com:2181:/hbase"
|
||||||
|
remote_wal_dir = "hdfs://srv1:9999/hbase"
|
||||||
|
table_cfs = { "ns3:table1" => [], "ns3:table2" => [],
|
||||||
|
"ns3:table3" => [] }
|
||||||
|
# add a new replication peer which serial flag is true
|
||||||
|
args = { CLUSTER_KEY => cluster_key, SERIAL => true,
|
||||||
|
TABLE_CFS => table_cfs}
|
||||||
|
command(:add_peer, @peer_id, args)
|
||||||
|
|
||||||
|
assert_equal(1, command(:list_peers).length)
|
||||||
|
peer = command(:list_peers).get(0)
|
||||||
|
assert_equal(@peer_id, peer.getPeerId)
|
||||||
|
assert_equal(cluster_key, peer.getPeerConfig.getClusterKey)
|
||||||
|
assert_equal(true, peer.getPeerConfig.isSerial)
|
||||||
|
assert_tablecfs_equal(table_cfs, peer.getPeerConfig.getTableCFsMap())
|
||||||
|
|
||||||
|
# cleanup for future tests
|
||||||
|
command(:remove_peer, @peer_id)
|
||||||
|
end
|
||||||
|
|
||||||
define_test "add_peer: remote wal dir" do
|
define_test "add_peer: remote wal dir" do
|
||||||
cluster_key = "server1.cie.com:2181:/hbase"
|
cluster_key = "server1.cie.com:2181:/hbase"
|
||||||
remote_wal_dir = "hdfs://srv1:9999/hbase"
|
remote_wal_dir = "hdfs://srv1:9999/hbase"
|
||||||
|
@ -490,6 +511,7 @@ module Hbase
|
||||||
|
|
||||||
assert_equal(1, command(:list_peers).length)
|
assert_equal(1, command(:list_peers).length)
|
||||||
peer_config = command(:list_peers).get(0).getPeerConfig
|
peer_config = command(:list_peers).get(0).getPeerConfig
|
||||||
|
# the default serial flag is false
|
||||||
assert_equal(false, peer_config.isSerial)
|
assert_equal(false, peer_config.isSerial)
|
||||||
|
|
||||||
command(:set_peer_serial, @peer_id, true)
|
command(:set_peer_serial, @peer_id, true)
|
||||||
|
|
|
@ -1898,7 +1898,28 @@ This treatment can possibly lead to data inconsistency between source and destin
|
||||||
|
|
||||||
.Serial replication configuration
|
.Serial replication configuration
|
||||||
|
|
||||||
. Set the serial flag to true for a repliation peer. You can either set it to true when creating a replication peer, or change it to true later.
|
Set the serial flag to true for a repliation peer. And the default serial flag is false.
|
||||||
|
|
||||||
|
* Add a new replication peer which serial flag is true
|
||||||
|
|
||||||
|
[source,ruby]
|
||||||
|
----
|
||||||
|
hbase> add_peer '1', CLUSTER_KEY => "server1.cie.com:2181:/hbase", SERIAL => true
|
||||||
|
----
|
||||||
|
|
||||||
|
* Set a replication peer's serial flag to false
|
||||||
|
|
||||||
|
[source,ruby]
|
||||||
|
----
|
||||||
|
hbase> set_peer_serial '1', false
|
||||||
|
----
|
||||||
|
|
||||||
|
* Set a replication peer's serial flag to true
|
||||||
|
|
||||||
|
[source,ruby]
|
||||||
|
----
|
||||||
|
hbase> set_peer_serial '1', true
|
||||||
|
----
|
||||||
|
|
||||||
The serial replication feature had been done firstly in link:https://issues.apache.org/jira/browse/HBASE-9465[HBASE-9465] and then reverted and redone in link:https://issues.apache.org/jira/browse/HBASE-20046[HBASE-20046]. You can find more details in these issues.
|
The serial replication feature had been done firstly in link:https://issues.apache.org/jira/browse/HBASE-9465[HBASE-9465] and then reverted and redone in link:https://issues.apache.org/jira/browse/HBASE-20046[HBASE-20046]. You can find more details in these issues.
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue