From 92fef40dbfc2b2e63147ac6ae1b30f707de42043 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 13 Nov 2018 06:45:00 -0500 Subject: [PATCH] Introduce CCR getting started guide (#35434) This commit introduces a basic getting started guide for cross-cluster replication to the docs. Co-authored-by: "lcawl " --- docs/reference/ccr/getting-started.asciidoc | 326 +++++++++++++++++++- 1 file changed, 324 insertions(+), 2 deletions(-) diff --git a/docs/reference/ccr/getting-started.asciidoc b/docs/reference/ccr/getting-started.asciidoc index a0f97a659f2..a0721e7f989 100644 --- a/docs/reference/ccr/getting-started.asciidoc +++ b/docs/reference/ccr/getting-started.asciidoc @@ -1,7 +1,329 @@ [role="xpack"] [testenv="platinum"] [[ccr-getting-started]] -== Getting Started +== Getting Started with {ccr} beta[] -This is the getting started section of the {ccr} docs. \ No newline at end of file + +This getting-started guide for {ccr} shows you how to: + +* <> +* <> in a remote cluster +* <> that replicates + a leader index +* <> + +[float] +[[ccr-getting-started-before-you-begin]] +=== Before you begin +. {stack-gs}/get-started-elastic-stack.html#install-elasticsearch[Install {es}] + on your local and remote clusters. + +. Obtain a license that includes the {ccr} features. See + https://www.elastic.co/subscriptions[subscriptions] and + <>. + +. If the Elastic {security-features} are enabled in your local and remote + clusters, you need a user that has appropriate authority to perform the steps + in this tutorial. ++ +-- +[[ccr-getting-started-security]] +The {ccr} features use cluster privileges and built-in roles to make it easier +to control which users have authority to manage {ccr}. + +By default, you can perform all of the steps in this tutorial by +using the built-in `elastic` user. However, a password must be set for this user +before the user can do anything. For information about how to set that password, +see <>. + +If you are performing these steps in a production environment, take extra care +because the `elastic` user has the `superuser` role and you could inadvertently +make significant changes. + +Alternatively, you can assign the appropriate privileges to a user ID of your +choice. On the remote cluster that contains the leader index, a user will need +the `read_ccr` cluster privilege and `monitor` and `read` privileges on the +leader index. + +[source,yml] +-------------------------------------------------- +ccr_user: + cluster: + - read_ccr + indices: + - names: [ 'leader-index' ] + privileges: + - monitor + - read +-------------------------------------------------- + +On the local cluster that contains the follower index, the same user will need +the `manage_ccr` cluster privilege and `monitor`, `read`, `write` and +`manage_follow_index` privileges on the follower index. + +[source,yml] +-------------------------------------------------- +ccr_user: + cluster: + - manage_ccr + indices: + - names: [ 'follower-index' ] + privileges: + - monitor + - read + - write + - manage_follow_index +-------------------------------------------------- + +If you are managing +<> via the +cluster update settings API, you will also need a user with the `all` cluster +privilege. +-- + +[float] +[[ccr-getting-started-remote-cluster]] +=== Connecting to a remote cluster + +The {ccr} features require that you +{ref}/modules-remote-clusters.html[connect your local cluster to a remote +cluster]. In this tutorial, we will connect our local cluster to a remote +cluster with the cluster alias `leader`. + +[source,js] +-------------------------------------------------- +PUT /_cluster/settings +{ + "persistent" : { + "cluster" : { + "remote" : { + "leader" : { + "seeds" : [ + "127.0.0.1:9300" <1> + ] + } + } + } + } +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:host] +// TEST[s/127.0.0.1:9300/\${transport_host}/] +<1> Specifies the hostname and transport port of a seed node in the remote + cluster. + +You can verify that the local cluster is successfully connected to the remote +cluster. + +[source,js] +-------------------------------------------------- +GET /_remote/info +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +The API will respond by showing that the local cluster is connected to the +remote cluster. + +[source,js] +-------------------------------------------------- +{ + "leader" : { + "seeds" : [ + "127.0.0.1:9300" + ], + "connected" : true, <1> + "num_nodes_connected" : 1, <2> + "max_connections_per_cluster" : 3, + "initial_connect_timeout" : "30s", + "skip_unavailable" : false + } +} +-------------------------------------------------- +// TESTRESPONSE +// TEST[s/127.0.0.1:9300/$body.leader.seeds.0/] +// TEST[s/"connected" : true/"connected" : $body.leader.connected/] +// TEST[s/"num_nodes_connected" : 1/"num_nodes_connected" : $body.leader.num_nodes_connected/] +<1> This shows the local cluster is connected to the remote cluster with cluster + alias `leader` +<2> This shows the number of nodes in the remote cluster the local cluster is + connected to. + +[float] +[[ccr-getting-started-leader-index]] +=== Creating a leader index + +Leader indices require special index settings to ensure that the operations that +need to be replicated are available when the +follower requests them from the leader. These settings are used to enable soft +deletes on the leader index and to control how many soft deletes are retained. A +_soft delete_ occurs whenever a document is deleted or updated. Soft deletes can +be enabled only on new indices created on or after {es} 6.5.0. + +In the following example, we will create a leader index in the remote cluster: + +[source,js] +-------------------------------------------------- +PUT /server-metrics +{ + "settings" : { + "index" : { + "number_of_shards" : 1, + "number_of_replicas" : 0, + "soft_deletes" : { + "enabled" : true, <1> + "retention" : { + "operations" : 1024 <2> + } + } + } + }, + "mappings" : { + "metric" : { + "properties" : { + "@timestamp" : { + "type" : "date" + }, + "accept" : { + "type" : "long" + }, + "deny" : { + "type" : "long" + }, + "host" : { + "type" : "keyword" + }, + "response" : { + "type" : "float" + }, + "service" : { + "type" : "keyword" + }, + "total" : { + "type" : "long" + } + } + } + } +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] +<1> Enables soft deletes on the leader index. +<2> Sets that up to 1024 soft deletes will be retained. + +[float] +[[ccr-getting-started-follower-index]] +=== Creating a follower index + +Follower indices are created with the {ref}/ccr-put-follow.html[create follower +API]. When you create a follower index, you must reference the +<> and the +<> that you created in the remote +cluster. + +[source,js] +-------------------------------------------------- +PUT /server-metrics-copy/_ccr/follow +{ + "remote_cluster" : "leader", + "leader_index" : "server-metrics" +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +////////////////////////// + +[source,js] +-------------------------------------------------- +{ + "follow_index_created" : true, + "follow_index_shards_acked" : true, + "index_following_started" : true +} +-------------------------------------------------- +// TESTRESPONSE + +////////////////////////// + +Now when you index documents into your leader index, you will see these +documents replicated in the follower index. You can +inspect the status of replication using the +{ref}/ccr-get-follow-stats[get follower stats API]. + +////////////////////////// + +[source,js] +-------------------------------------------------- +POST /server-metrics-copy/_ccr/pause_follow + +POST /server-metrics-copy/_close + +POST /server-metrics-copy/_ccr/unfollow +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +////////////////////////// + +[float] +[[ccr-getting-started-auto-follow]] +=== Automatically create follower indices + +The auto-follow feature in {ccr} helps for time series use cases where you want +to follow new indices that are periodically created in the remote cluster +(such as daily Beats indices). Auto-following is configured using the +{ref}/ccr-put-auto-follow-pattern.html[create auto-follow pattern API]. With an +auto-follow pattern, you reference the +<> that you connected your +local cluster to. You must also specify a collection of patterns that match the +indices you want to automatically follow. + +For example: + +[source,js] +-------------------------------------------------- +PUT /_ccr/auto_follow/beats +{ + "remote_cluster" : "leader", + "leader_index_patterns" : + [ + "metricbeat-*", <1> + "packetbeat-*" <2> + ], + "follow_index_pattern" : "{{leader_index}}-copy" <3> +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] +<1> Automatically follow new {metricbeat} indices. +<2> Automatically follow new {packetbeat} indices. +<3> The name of the follower index is derived from the name of the leader index + by adding the suffix `-copy` to the name of the leader index. + +////////////////////////// + +[source,js] +-------------------------------------------------- +{ + "acknowledged" : true +} +-------------------------------------------------- +// TESTRESPONSE + +////////////////////////// + +////////////////////////// + +[source,js] +-------------------------------------------------- +DELETE /_ccr/auto_follow/beats +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +//////////////////////////