From fdfdbe486d9e31530a483dcec3c77bccf7f398b4 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Fri, 26 Oct 2018 11:23:35 -0400 Subject: [PATCH] Introduce cross-cluster replication API docs (#34726) This commit is our first introduction to cross-cluster replication docs. In this commit, we introduce the cross-cluster replication API docs. We also add skelton docs for additional content that will be added in a series of follow-up commits. --- docs/build.gradle | 18 +- .../delete-auto-follow-pattern.asciidoc | 68 +++++ .../get-auto-follow-pattern.asciidoc | 93 +++++++ .../get-auto-follow-stats.asciidoc | 46 ++++ .../put-auto-follow-pattern.asciidoc | 118 ++++++++ docs/reference/ccr/apis/ccr-apis.asciidoc | 38 +++ .../ccr/apis/follow-request-body.asciidoc | 44 +++ .../ccr/apis/follow/get-follow-stats.asciidoc | 254 ++++++++++++++++++ .../apis/follow/post-pause-follow.asciidoc | 68 +++++ .../apis/follow/post-resume-follow.asciidoc | 94 +++++++ .../ccr/apis/follow/post-unfollow.asciidoc | 75 ++++++ .../ccr/apis/follow/put-follow.asciidoc | 94 +++++++ docs/reference/ccr/getting-started.asciidoc | 6 + docs/reference/ccr/index.asciidoc | 15 ++ docs/reference/ccr/overview.asciidoc | 6 + docs/reference/index.asciidoc | 2 + docs/reference/rest-api/index.asciidoc | 2 + 17 files changed, 1040 insertions(+), 1 deletion(-) create mode 100644 docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc create mode 100644 docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc create mode 100644 docs/reference/ccr/apis/auto-follow/get-auto-follow-stats.asciidoc create mode 100644 docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc create mode 100644 docs/reference/ccr/apis/ccr-apis.asciidoc create mode 100644 docs/reference/ccr/apis/follow-request-body.asciidoc create mode 100644 docs/reference/ccr/apis/follow/get-follow-stats.asciidoc create mode 100644 docs/reference/ccr/apis/follow/post-pause-follow.asciidoc create mode 100644 docs/reference/ccr/apis/follow/post-resume-follow.asciidoc create mode 100644 docs/reference/ccr/apis/follow/post-unfollow.asciidoc create mode 100644 docs/reference/ccr/apis/follow/put-follow.asciidoc create mode 100644 docs/reference/ccr/getting-started.asciidoc create mode 100644 docs/reference/ccr/index.asciidoc create mode 100644 docs/reference/ccr/overview.asciidoc diff --git a/docs/build.gradle b/docs/build.gradle index ce560e1ca42..99f82d95b58 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -139,7 +139,6 @@ buildRestTests.setups['host'] = ''' - do: nodes.info: metric: [ http, transport ] - - is_true: nodes.$master.http.publish_address - set: {nodes.$master.http.publish_address: host} - set: {nodes.$master.transport.publish_address: transport_host} ''' @@ -1083,4 +1082,21 @@ buildRestTests.setups['calendar_outages_addevent'] = buildRestTests.setups['cale ]} ''' +buildRestTests.setups['remote_cluster'] = buildRestTests.setups['host'] + ''' + - do: + cluster.put_settings: + body: + persistent: + cluster.remote.remote_cluster.seeds: $transport_host +''' +buildRestTests.setups['remote_cluster_and_leader_index'] = buildRestTests.setups['remote_cluster'] + ''' + - do: + indices.create: + index: leader_index + body: + settings: + index.number_of_replicas: 0 + index.number_of_shards: 1 + index.soft_deletes.enabled: true +''' diff --git a/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc new file mode 100644 index 00000000000..301c7f7da49 --- /dev/null +++ b/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc @@ -0,0 +1,68 @@ +[role="xpack"] +[testenv="platinum"] +[[ccr-delete-auto-follow-pattern]] +=== Delete Auto-Follow Pattern API +++++ +Delete Auto-Follow Pattern +++++ + +Delete auto-follow patterns. + +==== Description + +This API deletes a configured auto-follow pattern collection. + +==== Request + +////////////////////////// + +[source,js] +-------------------------------------------------- +PUT /_ccr/auto_follow/my_auto_follow_pattern +{ + "remote_cluster" : "remote_cluster", + "leader_index_patterns" : + [ + "leader_index" + ], + "follow_index_pattern" : "{{leader_index}}-follower" +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:remote_cluster] +// TESTSETUP + +////////////////////////// + +[source,js] +-------------------------------------------------- +DELETE /_ccr/auto_follow/ +-------------------------------------------------- +// CONSOLE +// TEST[s//my_auto_follow_pattern/] + +==== Path Parameters +`auto_follow_pattern_name` (required):: + (string) specifies the auto-follow pattern collection to delete + +==== Example + +This example deletes an auto-follow pattern collection named +`my_auto_follow_pattern`: + +[source,js] +-------------------------------------------------- +DELETE /_ccr/auto_follow/my_auto_follow_pattern +-------------------------------------------------- +// CONSOLE +// TEST[setup:remote_cluster] + +The API returns the following result: + +[source,js] +-------------------------------------------------- +{ + "acknowledged" : true +} +-------------------------------------------------- +// TESTRESPONSE diff --git a/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc new file mode 100644 index 00000000000..1ff9c9943c9 --- /dev/null +++ b/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc @@ -0,0 +1,93 @@ +[role="xpack"] +[testenv="platinum"] +[[ccr-get-auto-follow-pattern]] +=== Get Auto-Follow Pattern API +++++ +Get Auto-Follow Pattern +++++ + +Get auto-follow patterns. + +==== Description + +This API gets configured auto-follow patterns. This API will return the +specified auto-follow pattern collection. + +==== Request + +////////////////////////// + +[source,js] +-------------------------------------------------- +PUT /_ccr/auto_follow/my_auto_follow_pattern +{ + "remote_cluster" : "remote_cluster", + "leader_index_patterns" : + [ + "leader_index*" + ], + "follow_index_pattern" : "{{leader_index}}-follower" +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:remote_cluster] +// TESTSETUP + +[source,js] +-------------------------------------------------- +DELETE /_ccr/auto_follow/my_auto_follow_pattern +-------------------------------------------------- +// CONSOLE +// TEST +// TEARDOWN + +////////////////////////// + +[source,js] +-------------------------------------------------- +GET /_ccr/auto_follow/ +-------------------------------------------------- +// CONSOLE + +[source,js] +-------------------------------------------------- +GET /_ccr/auto_follow/ +-------------------------------------------------- +// CONSOLE +// TEST[s//my_auto_follow_pattern/] + +==== Path Parameters +`auto_follow_pattern_name`:: + (string) specifies the auto-follow pattern collection that you want to + retrieve; if you do not specify a name, the API returns information for all + collections + +==== Example + +This example retrieves information about an auto-follow pattern collection +named `my_auto_follow_pattern`: + +[source,js] +-------------------------------------------------- +GET /_ccr/auto_follow/my_auto_follow_pattern +-------------------------------------------------- +// CONSOLE +// TEST[setup:remote_cluster] + +The API returns the following result: + +[source,js] +-------------------------------------------------- +{ + "my_auto_follow_pattern" : + { + "remote_cluster" : "remote_cluster", + "leader_index_patterns" : + [ + "leader_index*" + ], + "follow_index_pattern" : "{{leader_index}}-follower" + } +} +-------------------------------------------------- +// TESTRESPONSE diff --git a/docs/reference/ccr/apis/auto-follow/get-auto-follow-stats.asciidoc b/docs/reference/ccr/apis/auto-follow/get-auto-follow-stats.asciidoc new file mode 100644 index 00000000000..85c6775af1c --- /dev/null +++ b/docs/reference/ccr/apis/auto-follow/get-auto-follow-stats.asciidoc @@ -0,0 +1,46 @@ +[role="xpack"] +[testenv="platinum"] +[[ccr-get-auto-follow-stats]] +=== Get Auto-Follow Stats API +++++ +Get Auto-Follow Stats +++++ + +Get auto-follow stats. + +==== Description + +This API gets stats about auto-follow patterns. + +==== Request + +[source,js] +-------------------------------------------------- +GET /_ccr/auto_follow/stats +-------------------------------------------------- +// CONSOLE +// TEST + +==== Example + +This example retrieves stats about auto-follow patterns: + +[source,js] +-------------------------------------------------- +GET /_ccr/auto_follow/stats +-------------------------------------------------- +// CONSOLE +// TEST + +The API returns the following result: + +[source,js] +-------------------------------------------------- +{ + "number_of_successful_follow_indices" : 16, + "number_of_failed_follow_indices" : 0, + "number_of_failed_remote_cluster_state_requests" : 0, + "recent_auto_follow_errors" : [ ] +} +-------------------------------------------------- +// TESTRESPONSE[s/"number_of_successful_follow_indices" : 16/"number_of_successful_follow_indices" : $body.number_of_successful_follow_indices/] diff --git a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc new file mode 100644 index 00000000000..e18b69579d3 --- /dev/null +++ b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc @@ -0,0 +1,118 @@ +[role="xpack"] +[testenv="platinum"] +[[ccr-put-auto-follow-pattern]] +=== Create Auto-Follow Pattern API +++++ +Create Auto-Follow Pattern +++++ + +Creates an auto-follow pattern. + +==== Description + +This API creates a new named collection of auto-follow patterns against the +remote cluster specified in the request body. Newly created indices on the +remote cluster matching any of the specified patterns will be automatically +configured as follower indices. + +==== Request + +[source,js] +-------------------------------------------------- +PUT /_ccr/auto_follow/ +{ + "remote_cluster" : "", + "leader_index_patterns" : + [ + "" + ], + "follow_index_pattern" : "" +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:remote_cluster] +// TEST[s//auto_follow_pattern_name/] +// TEST[s//remote_cluster/] +// TEST[s//leader_index*/] +// TEST[s//{{leader_index}}-follower/] + +////////////////////////// + +[source,js] +-------------------------------------------------- +DELETE /_ccr/auto_follow/auto_follow_pattern_name +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +////////////////////////// + +==== Path Parameters +`auto_follow_pattern_name` (required):: + (string) name of the collection of auto-follow patterns + +==== Request Body +`remote_cluster`:: + (required string) the <> containing the + leader indices to match against + +`leader_index_patterns`:: + (array) an array of simple index patterns to match against indices in the + remote cluster specified by the `remote_cluster` field + +`follow_index_pattern`:: + (string) the name of follower index; the template `{{leader_index}}` can be + used to derive the name of the follower index from the name of the leader + index + +include::../follow-request-body.asciidoc[] + +==== Example + +This example creates an auto-follow pattern named `my_auto_follow_pattern`: + +[source,js] +-------------------------------------------------- +PUT /_ccr/auto_follow/my_auto_follow_pattern +{ + "remote_cluster" : "remote_cluster", + "leader_index_patterns" : + [ + "leader_index*" + ], + "follow_index_pattern" : "{{leader_index}}-follower", + "max_read_request_operation_count" : 1024, + "max_outstanding_read_requests" : 16, + "max_read_request_size" : "1024k", + "max_write_request_operation_count" : 32768, + "max_write_request_size" : "16k", + "max_outstanding_write_requests" : 8, + "max_write_buffer_count" : 512, + "max_write_buffer_size" : "512k", + "max_retry_delay" : "10s", + "read_poll_timeout" : "30s" +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:remote_cluster] + +The API returns the following result: + +[source,js] +-------------------------------------------------- +{ + "acknowledged" : true +} +-------------------------------------------------- +// TESTRESPONSE + +////////////////////////// + +[source,js] +-------------------------------------------------- +DELETE /_ccr/auto_follow/my_auto_follow_pattern +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +////////////////////////// diff --git a/docs/reference/ccr/apis/ccr-apis.asciidoc b/docs/reference/ccr/apis/ccr-apis.asciidoc new file mode 100644 index 00000000000..d4a45bab6ed --- /dev/null +++ b/docs/reference/ccr/apis/ccr-apis.asciidoc @@ -0,0 +1,38 @@ +[role="xpack"] +[testenv="platinum"] +[[ccr-apis]] +== Cross-cluster replication APIs + +You can use the following APIs to perform {ccr} operations. + +[float] +[[ccr-api-follow]] +=== Follow + +* <> +* <> +* <> +* <> +* <> + +[float] +[[ccr-api-auto-follow]] +=== Auto-follow + +* <> +* <> +* <> +* <> + +// follow +include::follow/put-follow.asciidoc[] +include::follow/post-pause-follow.asciidoc[] +include::follow/post-resume-follow.asciidoc[] +include::follow/post-unfollow.asciidoc[] +include::follow/get-follow-stats.asciidoc[] + +// auto-follow +include::auto-follow/put-auto-follow-pattern.asciidoc[] +include::auto-follow/delete-auto-follow-pattern.asciidoc[] +include::auto-follow/get-auto-follow-pattern.asciidoc[] +include::auto-follow/get-auto-follow-stats.asciidoc[] diff --git a/docs/reference/ccr/apis/follow-request-body.asciidoc b/docs/reference/ccr/apis/follow-request-body.asciidoc new file mode 100644 index 00000000000..7215cc01302 --- /dev/null +++ b/docs/reference/ccr/apis/follow-request-body.asciidoc @@ -0,0 +1,44 @@ +`max_read_request_operation_count`:: + (integer) the maximum number of operations to pull per read from the remote + cluster + +`max_outstanding_read_requests`:: + (long) the maximum number of outstanding reads requests from the remote + cluster + +`max_read_request_size`:: + (<>) the maximum size in bytes of per read of a batch + of operations pulled from the remote cluster + +`max_write_request_operation_count`:: + (integer) the maximum number of operations per bulk write request executed on + the follower + +`max_write_request_size`:: + (<>) the maximum total bytes of operations per bulk write request + executed on the follower + +`max_outstanding_write_requests`:: + (integer) the maximum number of outstanding write requests on the follower + +`max_write_buffer_count`:: + (integer) the maximum number of operations that can be queued for writing; + when this limit is reached, reads from the remote cluster will be deferred + until the number of queued operations goes below the limit + +`max_write_buffer_size`:: + (<>) the maximum total bytes of operations that can be queued for + writing; when this limit is reached, reads from the remote cluster will be + deferred until the total bytes of queued operations goes below the limit + +`max_retry_delay`:: + (<>) the maximum time to wait before retrying an + operation that failed exceptionally; an exponential backoff strategy is + employed when retrying + +`read_poll_timeout`:: + (<>) the maximum time to wait for new operations on the + remote cluster when the follower index is synchronized with the leader index; + when the timeout has elapsed, the poll for operations will return to the + follower so that it can update some statistics, and then the follower will + immediately attempt to read from the leader again \ No newline at end of file diff --git a/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc b/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc new file mode 100644 index 00000000000..efbaeecb712 --- /dev/null +++ b/docs/reference/ccr/apis/follow/get-follow-stats.asciidoc @@ -0,0 +1,254 @@ +[role="xpack"] +[testenv="platinum"] +[[ccr-get-follow-stats]] +=== Get Follower Stats API +++++ +Get Follower Stats +++++ + +Get follower stats. + +==== Description + +This API gets follower stats. This API will return shard-level stats about the +following tasks associated with each shard for the specified indices. + +==== Request + +////////////////////////// + +[source,js] +-------------------------------------------------- +PUT /follower_index/_ccr/follow +{ + "remote_cluster" : "remote_cluster", + "leader_index" : "leader_index" +} +-------------------------------------------------- +// CONSOLE +// TESTSETUP +// TEST[setup:remote_cluster_and_leader_index] + +[source,js] +-------------------------------------------------- +POST /follower_index/_ccr/pause_follow +-------------------------------------------------- +// CONSOLE +// TEARDOWN + +////////////////////////// + +[source,js] +-------------------------------------------------- +GET /_ccr/stats +-------------------------------------------------- +// CONSOLE + +[source,js] +-------------------------------------------------- +GET //_ccr/stats +-------------------------------------------------- +// CONSOLE +// TEST[s//follower_index/] + +==== Path Parmeters +`index` :: + (string) a comma-delimited list of index patterns + +==== Results + +This API returns the following information: + +`indices`:: + (array) an array of follower index statistics + +The `indices` array consists of objects containing two fields: + +`indices[].index`:: + (string) the name of the follower index + +`indices[].shards`:: + (array) an array of shard-level following task statistics + +The `shards` array consists of objects containing the following fields: + +`indices[].shards[].remote_cluster`:: + (string) the > containing the leader + index + +`indices[].shards[].leader_index`:: + (string) the name of the index in the leader cluster being followed + +`indices[].shards[].follower_index`:: + (string) the name of the follower index + +`indices[].shards[].shard_id`:: + (integer) the numerical shard ID, with values from 0 to one less than the + number of replicas + +`indices[].shards[].leader_global_checkpoint`:: + (long) the current global checkpoint on the leader known to the follower task + +`indices[].shards[].leader_max_seq_no`:: + (long) the current maximum sequence number on the leader known to the follower + task + +`indices[].shards[].follower_global_checkpoint`:: + (long) the current global checkpoint on the follower; the difference between the + `leader_global_checkpoint` and the `follower_global_checkpoint` is an + indication of how much the follower is lagging the leader + +`indices[].shards[].follower_max_seq_no`:: + (long) the current maximum sequence number on the follower + +`indices[].shards[].last_requested_seq_no`:: + (long) the starting sequence number of the last batch of operations requested + from the leader + +`indices[].shards[].outstanding_read_requests`:: + (integer) the number of active read requests from the follower + +`indices[].shards[].outstanding_write_requests`:: + (integer) the number of active bulk write requests on the follower + +`indices[].shards[].write_buffer_operation_count`:: + (integer) the number of write operations queued on the follower + +`indices[].shards[].follower_mapping_version`:: + (long) the mapping version the follower is synced up to + +`indices[].shards[].total_read_time_millis`:: + (long) the total time reads were outstanding, measured from the time a read + was sent to the leader to the time a reply was returned to the follower + +`indices[].shards[].total_read_remote_exec_time_millis`:: + (long) the total time reads spent executing on the remote cluster + +`indices[].shards[].successful_read_requests`:: + (long) the number of successful fetches + +`indices[].shards[].failed_read_requests`:: + (long) the number of failed reads + +`indices[].shards[].operations_read`:: + (long) the total number of operations read from the leader + +`indices[].shards[].bytes_read`:: + (long) the total of transferred bytes read from the leader (note this is only + an estimate, and does not account for compression if enabled) + +`indices[].shards[].total_write_time_millis`:: + (long) the total time spent writing on the follower + +`indices[].shards[].write_buffer_size_in_bytes`:: + (long) the total number of bytes of operations currently queued for writing + +`indices[].shards[].successful_write_requests`:: + (long) the number of bulk write requests executed on the follower + +`indices[].shards[].failed_write_requests`:: + (long) the number of failed bulk write requests executed on the follower + +`indices[].shards[].operations_written`:: + (long) the number of operations written on the follower + +`indices[].shards[].read_exceptions`:: + (array) an array of objects representing failed reads + +The `read_exceptions` array consists of objects containing the following +fields: + +`indices[].shards[].read_exceptions[].from_seq_no`:: + (long) the starting sequence number of the batch requested from the leader + +`indices[].shards[].read_exceptions[].retries`:: + (integer) the number of times the batch has been retried + +`indices[].shards[].read_exceptions[].exception`:: + (object) represents the exception that caused the read to fail + +Continuing with the fields from `shards`: + +`indices[].shards[].time_since_last_read_millis`:: + (long) the number of milliseconds since a read request was sent to the leader; + note that when the follower is caught up to the leader, this number will + increase up to the configured `read_poll_timeout` at which point another read + request will be sent to the leader + +`indices[].fatal_exception`:: + (object) an object representing a fatal exception that cancelled the following + task; in this situation, the following task must be resumed manually with the + <> + +==== Example + +This example retrieves follower stats: + +[source,js] +-------------------------------------------------- +GET /_ccr/stats +-------------------------------------------------- +// CONSOLE + +The API returns the following results: +[source,js] +-------------------------------------------------- +{ + "indices" : [ + { + "index" : "follower_index", + "shards" : [ + { + "remote_cluster" : "remote_cluster", + "leader_index" : "leader_index", + "follower_index" : "follower_index", + "shard_id" : 0, + "leader_global_checkpoint" : 1024, + "leader_max_seq_no" : 1536, + "follower_global_checkpoint" : 768, + "follower_max_seq_no" : 896, + "last_requested_seq_no" : 897, + "outstanding_read_requests" : 8, + "outstanding_write_requests" : 2, + "write_buffer_operation_count" : 64, + "follower_mapping_version" : 4, + "total_read_time_millis" : 32768, + "total_read_remote_exec_time_millis" : 16384, + "successful_read_requests" : 32, + "failed_read_requests" : 0, + "operations_read" : 896, + "bytes_read" : 32768, + "total_write_time_millis" : 16384, + "write_buffer_size_in_bytes" : 1536, + "successful_write_requests" : 16, + "failed_write_requests" : 0, + "operations_written" : 832, + "read_exceptions" : [ ], + "time_since_last_read_millis" : 8 + } + ] + } + ] +} +-------------------------------------------------- +// TESTRESPONSE[s/"leader_global_checkpoint" : 1024/"leader_global_checkpoint" : $body.indices.0.shards.0.leader_global_checkpoint/] +// TESTRESPONSE[s/"leader_max_seq_no" : 1536/"leader_max_seq_no" : $body.indices.0.shards.0.leader_max_seq_no/] +// TESTRESPONSE[s/"follower_global_checkpoint" : 768/"follower_global_checkpoint" : $body.indices.0.shards.0.follower_global_checkpoint/] +// TESTRESPONSE[s/"follower_max_seq_no" : 896/"follower_max_seq_no" : $body.indices.0.shards.0.follower_max_seq_no/] +// TESTRESPONSE[s/"last_requested_seq_no" : 897/"last_requested_seq_no" : $body.indices.0.shards.0.last_requested_seq_no/] +// TESTRESPONSE[s/"outstanding_read_requests" : 8/"outstanding_read_requests" : $body.indices.0.shards.0.outstanding_read_requests/] +// TESTRESPONSE[s/"outstanding_write_requests" : 2/"outstanding_write_requests" : $body.indices.0.shards.0.outstanding_write_requests/] +// TESTRESPONSE[s/"write_buffer_operation_count" : 64/"write_buffer_operation_count" : $body.indices.0.shards.0.write_buffer_operation_count/] +// TESTRESPONSE[s/"follower_mapping_version" : 4/"follower_mapping_version" : $body.indices.0.shards.0.follower_mapping_version/] +// TESTRESPONSE[s/"total_read_time_millis" : 32768/"total_read_time_millis" : $body.indices.0.shards.0.total_read_time_millis/] +// TESTRESPONSE[s/"total_read_remote_exec_time_millis" : 16384/"total_read_remote_exec_time_millis" : $body.indices.0.shards.0.total_read_remote_exec_time_millis/] +// TESTRESPONSE[s/"successful_read_requests" : 32/"successful_read_requests" : $body.indices.0.shards.0.successful_read_requests/] +// TESTRESPONSE[s/"failed_read_requests" : 0/"failed_read_requests" : $body.indices.0.shards.0.failed_read_requests/] +// TESTRESPONSE[s/"operations_read" : 896/"operations_read" : $body.indices.0.shards.0.operations_read/] +// TESTRESPONSE[s/"bytes_read" : 32768/"bytes_read" : $body.indices.0.shards.0.bytes_read/] +// TESTRESPONSE[s/"total_write_time_millis" : 16384/"total_write_time_millis" : $body.indices.0.shards.0.total_write_time_millis/] +// TESTRESPONSE[s/"write_buffer_size_in_bytes" : 1536/"write_buffer_size_in_bytes" : $body.indices.0.shards.0.write_buffer_size_in_bytes/] +// TESTRESPONSE[s/"successful_write_requests" : 16/"successful_write_requests" : $body.indices.0.shards.0.successful_write_requests/] +// TESTRESPONSE[s/"failed_write_requests" : 0/"failed_write_requests" : $body.indices.0.shards.0.failed_write_requests/] +// TESTRESPONSE[s/"operations_written" : 832/"operations_written" : $body.indices.0.shards.0.operations_written/] +// TESTRESPONSE[s/"time_since_last_read_millis" : 8/"time_since_last_read_millis" : $body.indices.0.shards.0.time_since_last_read_millis/] diff --git a/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc b/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc new file mode 100644 index 00000000000..7fa4dbdd455 --- /dev/null +++ b/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc @@ -0,0 +1,68 @@ +[role="xpack"] +[testenv="platinum"] +[[ccr-post-pause-follow]] +=== Pause Follower API +++++ +Pause Follower +++++ + +Pauses a follower index. + +==== Description + +This API pauses a follower index. When this API returns, the follower index will +not fetch any additional operations from the leader index. You can resume +following with the <>. Pausing and +resuming a follower index can be used to change the configuration of the +following task. + +==== Request + +////////////////////////// + +[source,js] +-------------------------------------------------- +PUT /follower_index/_ccr/follow +{ + "remote_cluster" : "remote_cluster", + "leader_index" : "leader_index" +} +-------------------------------------------------- +// CONSOLE +// TESTSETUP +// TEST[setup:remote_cluster_and_leader_index] + +////////////////////////// + +[source,js] +-------------------------------------------------- +POST //_ccr/pause_follow +-------------------------------------------------- +// CONSOLE +// TEST[s//follower_index/] + +==== Path Parameters + +`follower_index` (required):: + (string) the name of the follower index + +==== Example + +This example pauses a follower index named `follower_index`: + +[source,js] +-------------------------------------------------- +POST /follower_index/_ccr/pause_follow +-------------------------------------------------- +// CONSOLE +// TEST + +The API returns the following result: + +[source,js] +-------------------------------------------------- +{ + "acknowledged" : true +} +-------------------------------------------------- +// TESTRESPONSE diff --git a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc new file mode 100644 index 00000000000..eb19050961b --- /dev/null +++ b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc @@ -0,0 +1,94 @@ +[role="xpack"] +[testenv="platinum"] +[[ccr-post-resume-follow]] +=== Resume Follower API +++++ +Resume Follower +++++ + +Resumes a follower index. + +==== Description + +This API resumes a follower index that has been paused either explicitly with +the <> or implicitly due to +execution that can not be retried due to failure during following. When this API +returns, the follower index will resume fetching operations from the leader index. + +==== Request + +////////////////////////// + +[source,js] +-------------------------------------------------- +PUT /follower_index/_ccr/follow +{ + "remote_cluster" : "remote_cluster", + "leader_index" : "leader_index" +} + +POST /follower_index/_ccr/pause_follow +-------------------------------------------------- +// CONSOLE +// TESTSETUP +// TEST[setup:remote_cluster_and_leader_index] + +[source,js] +-------------------------------------------------- +POST /follower_index/_ccr/pause_follow +-------------------------------------------------- +// CONSOLE +// TEARDOWN + +////////////////////////// + +[source,js] +-------------------------------------------------- +POST //_ccr/resume_follow +{ +} +-------------------------------------------------- +// CONSOLE +// TEST[s//follower_index/] +// TEST[s//remote_cluster/] +// TEST[s//leader_index/] + +==== Path Parameters + +`follower_index` (required):: + (string) the name of the follower index + +==== Request Body +include::../follow-request-body.asciidoc[] + +==== Example + +This example resumes a follower index named `follower_index`: + +[source,js] +-------------------------------------------------- +POST /follower_index/_ccr/resume_follow +{ + "max_read_request_operation_count" : 1024, + "max_outstanding_read_requests" : 16, + "max_read_request_size" : "1024k", + "max_write_request_operation_count" : 32768, + "max_write_request_size" : "16k", + "max_outstanding_write_requests" : 8, + "max_write_buffer_count" : 512, + "max_write_buffer_size" : "512k", + "max_retry_delay" : "10s", + "read_poll_timeout" : "30s" +} +-------------------------------------------------- +// CONSOLE + +The API returns the following result: + +[source,js] +-------------------------------------------------- +{ + "acknowledged" : true +} +-------------------------------------------------- +// TESTRESPONSE diff --git a/docs/reference/ccr/apis/follow/post-unfollow.asciidoc b/docs/reference/ccr/apis/follow/post-unfollow.asciidoc new file mode 100644 index 00000000000..d84f1704179 --- /dev/null +++ b/docs/reference/ccr/apis/follow/post-unfollow.asciidoc @@ -0,0 +1,75 @@ +[role="xpack"] +[testenv="platinum"] +[[ccr-post-unfollow]] +=== Unfollow API +++++ +Unfollow +++++ + +Converts a follower index to a regular index. + +==== Description + +This API stops the following task associated with a follower index and removes +index metadata and settings associated with {ccr}. This enables the index to +treated as a regular index. The follower index must be paused and closed before +invoking the unfollow API. + +NOTE: Currently {ccr} does not support converting an existing regular index to a +follower index. Converting a follower index to a regular index is an +irreversible operation. + +==== Request + +////////////////////////// + +[source,js] +-------------------------------------------------- +PUT /follower_index/_ccr/follow +{ + "remote_cluster" : "remote_cluster", + "leader_index" : "leader_index" +} + +POST /follower_index/_ccr/pause_follow + +POST /follower_index/_close +-------------------------------------------------- +// CONSOLE +// TESTSETUP +// TEST[setup:remote_cluster_and_leader_index] + +////////////////////////// + +[source,js] +-------------------------------------------------- +POST //_ccr/unfollow +-------------------------------------------------- +// CONSOLE +// TEST[s//follower_index/] + +==== Path Parmeters + +`follower_index` (required):: + (string) the name of the follower index + +==== Example + +This example converts `follower_index` from a follower index to a regular index: + +[source,js] +-------------------------------------------------- +POST /follower_index/_ccr/unfollow +-------------------------------------------------- +// CONSOLE +// TEST + +The API returns the following result: + +[source,js] +-------------------------------------------------- +{ + "acknowledged" : true +} +-------------------------------------------------- +// TESTRESPONSE diff --git a/docs/reference/ccr/apis/follow/put-follow.asciidoc b/docs/reference/ccr/apis/follow/put-follow.asciidoc new file mode 100644 index 00000000000..db0005fe3c9 --- /dev/null +++ b/docs/reference/ccr/apis/follow/put-follow.asciidoc @@ -0,0 +1,94 @@ +[role="xpack"] +[testenv="platinum"] +[[ccr-put-follow]] +=== Create Follower API +++++ +Create Follower +++++ + +Creates a follower index. + +==== Description + +This API creates a new follower index that is configured to follow the +referenced leader index. When this API returns, the follower index exists, and +{ccr} starts replicating operations from the leader index to the follower index. + +==== Request + +////////////////////////// + +[source,js] +-------------------------------------------------- +POST /follower_index/_ccr/pause_follow +-------------------------------------------------- +// CONSOLE +// TEARDOWN + +////////////////////////// + +[source,js] +-------------------------------------------------- +PUT //_ccr/follow +{ + "remote_cluster" : "", + "leader_index" : "" +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:remote_cluster_and_leader_index] +// TEST[s//follower_index/] +// TEST[s//remote_cluster/] +// TEST[s//leader_index/] + +==== Path Parameters + +`follower_index` (required):: + (string) the name of the follower index + +==== Request Body +`remote_cluster` (required):: + (string) the <> containing the leader + index + +`leader_index` (required):: + (string) the name of the index in the leader cluster to follow + +include::../follow-request-body.asciidoc[] + +==== Example + +This example creates a follower index named `follower_index`: + +[source,js] +-------------------------------------------------- +PUT /follower_index/_ccr/follow +{ + "remote_cluster" : "remote_cluster", + "leader_index" : "leader_index", + "max_read_request_operation_count" : 1024, + "max_outstanding_read_requests" : 16, + "max_read_request_size" : "1024k", + "max_write_request_operation_count" : 32768, + "max_write_request_size" : "16k", + "max_outstanding_write_requests" : 8, + "max_write_buffer_count" : 512, + "max_write_buffer_size" : "512k", + "max_retry_delay" : "10s", + "read_poll_timeout" : "30s" +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:remote_cluster_and_leader_index] + +The API returns the following result: + +[source,js] +-------------------------------------------------- +{ + "follow_index_created" : true, + "follow_index_shards_acked" : true, + "index_following_started" : true +} +-------------------------------------------------- +// TESTRESPONSE diff --git a/docs/reference/ccr/getting-started.asciidoc b/docs/reference/ccr/getting-started.asciidoc new file mode 100644 index 00000000000..daa6f298e5f --- /dev/null +++ b/docs/reference/ccr/getting-started.asciidoc @@ -0,0 +1,6 @@ +[role="xpack"] +[testenv="platinum"] +[[ccr-getting-started]] +== Getting Started + +This is the getting started section of the {ccr} docs. \ No newline at end of file diff --git a/docs/reference/ccr/index.asciidoc b/docs/reference/ccr/index.asciidoc new file mode 100644 index 00000000000..1d5e9445a7b --- /dev/null +++ b/docs/reference/ccr/index.asciidoc @@ -0,0 +1,15 @@ +[role="xpack"] +[testenv="platinum"] +[[xpack-ccr]] += Cross-cluster replication + +[partintro] +-- + +* <> +* <> + +-- + +include::overview.asciidoc[] +include::getting-started.asciidoc[] diff --git a/docs/reference/ccr/overview.asciidoc b/docs/reference/ccr/overview.asciidoc new file mode 100644 index 00000000000..648a981bc5b --- /dev/null +++ b/docs/reference/ccr/overview.asciidoc @@ -0,0 +1,6 @@ +[role="xpack"] +[testenv="platinum"] +[[ccr-overview]] +== Overview + +This is the overview section of the {ccr} docs. diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index 216983bc6f0..dd841fccda7 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -55,6 +55,8 @@ include::index-modules.asciidoc[] include::ingest.asciidoc[] +include::ccr/index.asciidoc[] + include::sql/index.asciidoc[] include::monitoring/index.asciidoc[] diff --git a/docs/reference/rest-api/index.asciidoc b/docs/reference/rest-api/index.asciidoc index c6243ab2598..5aef27e1275 100644 --- a/docs/reference/rest-api/index.asciidoc +++ b/docs/reference/rest-api/index.asciidoc @@ -8,6 +8,7 @@ directly to configure and access {xpack} features. * <> +* <> * <> * <> * <> @@ -19,6 +20,7 @@ directly to configure and access {xpack} features. include::info.asciidoc[] +include::{es-repo-dir}/ccr/apis/ccr-apis.asciidoc[] include::{es-repo-dir}/graph/explore.asciidoc[] include::{es-repo-dir}/licensing/index.asciidoc[] include::{es-repo-dir}/migration/migration.asciidoc[]