diff --git a/FORMATTING_GUIDE.md b/FORMATTING_GUIDE.md index 9c6a6b7c..fc18d2fa 100644 --- a/FORMATTING_GUIDE.md +++ b/FORMATTING_GUIDE.md @@ -32,6 +32,8 @@ This guide provides an overview of the formatted elements commonly used in the O When adding a page or a section, make the `nav_order` of the child pages multiples of 10. For example, if you have a parent page `Clients`, make child pages `Java`, `Python`, and `JavaScript` have a `nav_order` of 10, 20, and 30, respectively. Doing so makes inserting additional child pages easier because it does not require you to renumber existing pages. +Each collection must have an `index.md` file that corresponds to the collection's index page. In the `index.md` file's front matter, specify `nav_excluded: true` so that the page does not appear separately under the collection. + ## Buttons You can use either `copy` or `copy-curl` includes for code snippets. The `copy` include places a **Copy** button on the code snippet, while the `copy-curl` include places both **Copy** and **Copy as cURL** buttons. Use the `copy-curl` include for API requests. If an API request is already in the cURL format, use the `copy` include. diff --git a/_api-reference/index.md b/_api-reference/index.md index cbd516ae..b53ad4ea 100644 --- a/_api-reference/index.md +++ b/_api-reference/index.md @@ -3,6 +3,7 @@ layout: default title: REST API reference nav_order: 1 has_toc: true +nav_exclude: true redirect_from: - /opensearch/rest-api/index/ --- diff --git a/_clients/index.md b/_clients/index.md index 6c6cc920..c12c4719 100644 --- a/_clients/index.md +++ b/_clients/index.md @@ -3,6 +3,7 @@ layout: default title: Language clients nav_order: 1 has_children: false +nav_exclude: true redirect_from: - /clients/ --- diff --git a/_dashboards/index.md b/_dashboards/index.md index 0970f3d3..9346374b 100644 --- a/_dashboards/index.md +++ b/_dashboards/index.md @@ -3,6 +3,7 @@ layout: default title: OpenSearch Dashboards nav_order: 1 has_children: false +nav_exclude: true --- # OpenSearch Dashboards diff --git a/_data-prepper/index.md b/_data-prepper/index.md index 20f27693..f22fdd1c 100644 --- a/_data-prepper/index.md +++ b/_data-prepper/index.md @@ -4,6 +4,7 @@ title: Data Prepper nav_order: 1 has_children: false has_toc: false +nav_exclude: true redirect_from: - /clients/data-prepper/index/ - /data-prepper/ diff --git a/_field-types/index.md b/_field-types/index.md index f6c31b84..27ef3ad4 100644 --- a/_field-types/index.md +++ b/_field-types/index.md @@ -2,6 +2,7 @@ layout: default title: Mappings and field types nav_order: 1 +nav_exclude: true redirect_from: - /opensearch/mappings/ --- diff --git a/_im-plugin/index.md b/_im-plugin/index.md index 181d8607..f912c41d 100644 --- a/_im-plugin/index.md +++ b/_im-plugin/index.md @@ -3,6 +3,7 @@ layout: default title: Managing indexes nav_order: 1 has_children: false +nav_exclude: true redirect_from: - /im-plugin/ - /opensearch/index-data/ diff --git a/_install-and-configure/index.md b/_install-and-configure/index.md index 9237d4d5..a61cc1b1 100644 --- a/_install-and-configure/index.md +++ b/_install-and-configure/index.md @@ -4,6 +4,7 @@ title: Install and upgrade OpenSearch nav_order: 1 has_children: false has_toc: false +nav_exclude: true --- # Install and upgrade OpenSearch diff --git a/_job-scheduler-plugin/index.md b/_job-scheduler-plugin/index.md deleted file mode 100644 index 028a02e0..00000000 --- a/_job-scheduler-plugin/index.md +++ /dev/null @@ -1,130 +0,0 @@ ---- -layout: default -title: Job Scheduler -nav_order: 1 -has_children: false -has_toc: false ---- - -# Job Scheduler - -The OpenSearch Job Scheduler plugin provides a framework that can be used to build schedules for common tasks performed on your cluster. You can use Job Scheduler’s Service Provider Interface (SPI) to define schedules for cluster management tasks such as taking snapshots, managing your data’s lifecycle, and running periodic jobs. Job Scheduler has a sweeper that listens for updated events on the OpenSearch cluster and a scheduler that manages when jobs run. - -You can install the Job Scheduler plugin by following the standard [OpenSearch plugin installation]({{site.url}}{{site.baseurl}}/install-and-configure/install-opensearch/plugins/) process. The sample-extension-plugin example provided in the [Job Scheduler GitHub repository](https://github.com/opensearch-project/job-scheduler) provides a complete example of utilizing Job Scheduler when building a plugin. To define schedules, you build a plugin that implements the interfaces provided in the Job Scheduler library. You can schedule jobs by specifying an interval, or you can use a Unix cron expression such as `0 12 * * ?`, which runs at noon every day, to define a more flexible schedule. - -## Building a plugin for Job Scheduler - -OpenSearch plugin developers can extend the Job Scheduler plugin to schedule jobs to perform on the cluster. Jobs you can schedule include running aggregation queries against raw data, saving the aggregated data to a new index every hour, or continuing to monitor the shard allocation by calling the OpenSearch API and then posting the output to a webhook. - -For examples of building a plugin that uses the Job Scheduler plugin, see the Job Scheduler [README](https://github.com/opensearch-project/job-scheduler/blob/main/README.md). - -## Defining an endpoint - -You can configure your plugin's API endpoint by referencing the [example](https://github.com/opensearch-project/job-scheduler/blob/main/sample-extension-plugin/src/main/java/org/opensearch/jobscheduler/sampleextension/SampleExtensionRestHandler.java) `SampleExtensionRestHandler.java` file. Set the endpoint URL that your plugin will expose with `WATCH_INDEX_URI`: - -```java -public class SampleExtensionRestHandler extends BaseRestHandler { - public static final String WATCH_INDEX_URI = "/_plugins/scheduler_sample/watch"; -``` - -You can define the job configuration by [extending](https://github.com/opensearch-project/job-scheduler/blob/main/sample-extension-plugin/src/main/java/org/opensearch/jobscheduler/sampleextension/SampleJobParameter.java) `ScheduledJobParameter`. You can also define the fields used by your plugin, like `indexToWatch`, as shown in the [example](https://github.com/opensearch-project/job-scheduler/blob/main/sample-extension-plugin/src/main/java/org/opensearch/jobscheduler/sampleextension/SampleJobParameter.java) `SampleJobParameter` file. This job configuration will be saved as a document in an index you define, as shown in [this example](https://github.com/opensearch-project/job-scheduler/blob/main/sample-extension-plugin/src/main/java/org/opensearch/jobscheduler/sampleextension/SampleExtensionPlugin.java#L54). - -## Configuring parameters - -You can configure your plugin's parameters by referencing the [example](https://github.com/opensearch-project/job-scheduler/blob/main/sample-extension-plugin/src/main/java/org/opensearch/jobscheduler/sampleextension/SampleJobParameter.java) `SampleJobParameter.java` file and modifying it to fit your needs: - -```java -/** - * A sample job parameter. - *

- * It adds an additional "indexToWatch" field to {@link ScheduledJobParameter}, which stores the index - * the job runner will watch. - */ -public class SampleJobParameter implements ScheduledJobParameter { - public static final String NAME_FIELD = "name"; - public static final String ENABLED_FILED = "enabled"; - public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; - public static final String LAST_UPDATE_TIME_FIELD_READABLE = "last_update_time_field"; - public static final String SCHEDULE_FIELD = "schedule"; - public static final String ENABLED_TIME_FILED = "enabled_time"; - public static final String ENABLED_TIME_FILED_READABLE = "enabled_time_field"; - public static final String INDEX_NAME_FIELD = "index_name_to_watch"; - public static final String LOCK_DURATION_SECONDS = "lock_duration_seconds"; - public static final String JITTER = "jitter"; - - private String jobName; - private Instant lastUpdateTime; - private Instant enabledTime; - private boolean isEnabled; - private Schedule schedule; - private String indexToWatch; - private Long lockDurationSeconds; - private Double jitter; -``` - -Next, configure the request parameters you would like your plugin to use with Job Scheduler. These will be based on the variables you declare when configuring your plugin. The following example shows the request parameters you set when building your plugin: - -```java -public SampleJobParameter(String id, String name, String indexToWatch, Schedule schedule, Long lockDurationSeconds, Double jitter) { - this.jobName = name; - this.indexToWatch = indexToWatch; - this.schedule = schedule; - - Instant now = Instant.now(); - this.isEnabled = true; - this.enabledTime = now; - this.lastUpdateTime = now; - this.lockDurationSeconds = lockDurationSeconds; - this.jitter = jitter; - } - - @Override - public String getName() { - return this.jobName; - } - - @Override - public Instant getLastUpdateTime() { - return this.lastUpdateTime; - } - - @Override - public Instant getEnabledTime() { - return this.enabledTime; - } - - @Override - public Schedule getSchedule() { - return this.schedule; - } - - @Override - public boolean isEnabled() { - return this.isEnabled; - } - - @Override - public Long getLockDurationSeconds() { - return this.lockDurationSeconds; - } - - @Override public Double getJitter() { - return jitter; - } -``` - -The following table describes the request parameters configured in the previous example. All the request parameters shown are required. - -| Field | Data type | Description | -:--- | :--- | :--- -| getName | String | Returns the name of the job. | -| getLastUpdateTime | Time unit | Returns the time that the job was last run. | -| getEnabledTime | Time unit | Returns the time that the job was enabled. | -| getSchedule | Unix cron | Returns the job schedule formatted in Unix cron syntax. | -| isEnabled | Boolean | Indicates whether or not the job is enabled. | -| getLockDurationSeconds | Integer | Returns the duration of time for which the job is locked. | -| getJitter | Integer | Returns the defined jitter value. | - -The logic used by your job should be defined by a class extended from `ScheduledJobRunner` in the `SampleJobParameter.java` sample file, such as `SampleJobRunner`. While the job is running, there is a locking mechanism you can use to prevent other nodes from running the same job. First, [acquire](https://github.com/opensearch-project/job-scheduler/blob/main/sample-extension-plugin/src/main/java/org/opensearch/jobscheduler/sampleextension/SampleJobRunner.java#L96) the lock. Then make sure to release the lock before the [job finishes](https://github.com/opensearch-project/job-scheduler/blob/main/sample-extension-plugin/src/main/java/org/opensearch/jobscheduler/sampleextension/SampleJobRunner.java#L116). - -For more information, see the Job Scheduler [sample extension](https://github.com/opensearch-project/job-scheduler/blob/main/sample-extension-plugin/src/main/java/org/opensearch/jobscheduler/sampleextension/SampleJobParameter.java) directory in the [Job Scheduler GitHub repo](https://github.com/opensearch-project/job-scheduler). diff --git a/_layouts/default.html b/_layouts/default.html index dda0586b..443a5f3a 100755 --- a/_layouts/default.html +++ b/_layouts/default.html @@ -82,12 +82,14 @@ layout: table_wrappers {%- if collection.size > 0 -%} {%- endif -%} -

{{ collection_value.name }}
+ {% assign collection_url_path = collection_key | append: "/index/" %} +
{{ collection_value.name }}
{% include nav.html pages=collection key=collection_key %} {% else %} -
{{ collection_value.name }}
+ {% assign collection_url_path = collection_key | append: "/index/" %} +
{{ collection_value.name }}
{% include nav.html pages=collection key=collection_key %} {% endif %} {% else %} diff --git a/_ml-commons-plugin/index.md b/_ml-commons-plugin/index.md index 244b85ea..2358b982 100644 --- a/_ml-commons-plugin/index.md +++ b/_ml-commons-plugin/index.md @@ -4,6 +4,7 @@ title: About ML Commons nav_order: 1 has_children: false has_toc: false +nav_exclude: true --- # ML Commons plugin diff --git a/_monitoring-your-cluster/index.md b/_monitoring-your-cluster/index.md index 92255382..f6c57485 100644 --- a/_monitoring-your-cluster/index.md +++ b/_monitoring-your-cluster/index.md @@ -4,6 +4,7 @@ title: Monitoring your cluster nav_order: 1 has_children: false has_toc: false +nav_exclude: true --- # Monitoring your cluster diff --git a/_observing-your-data/index.md b/_observing-your-data/index.md index ed640caa..22d2564e 100644 --- a/_observing-your-data/index.md +++ b/_observing-your-data/index.md @@ -3,6 +3,7 @@ layout: default title: Observability nav_order: 1 has_children: true +nav_exclude: true redirect_from: - /observability-plugin/index/ --- diff --git a/_query-dsl/index.md b/_query-dsl/index.md index 94c0f35f..9eb9aeeb 100644 --- a/_query-dsl/index.md +++ b/_query-dsl/index.md @@ -4,6 +4,7 @@ title: Query DSL, aggregations, and analyzers nav_order: 1 has_children: false has_toc: false +nav_exclude: true --- # Query DSL, aggregations, and analyzers diff --git a/_replication-plugin/api.md b/_replication-plugin/api.md deleted file mode 100644 index 2b832958..00000000 --- a/_replication-plugin/api.md +++ /dev/null @@ -1,391 +0,0 @@ ---- -layout: default -title: API -nav_order: 50 ---- - -# Cross-cluster replication API - -Use these replication operations to programmatically manage cross-cluster replication. - -#### Table of contents -- TOC -{:toc} - -## Start replication -Introduced 1.1 -{: .label .label-purple } - -Initiate replication of an index from the leader cluster to the follower cluster. Send this request to the follower cluster. - - -#### Request - -```json -PUT /_plugins/_replication//_start -{ - "leader_alias":"", - "leader_index":"", - "use_roles":{ - "leader_cluster_role":"", - "follower_cluster_role":"" - } -} -``` - -Specify the following options: - -Options | Description | Type | Required -:--- | :--- |:--- |:--- | -`leader_alias` | The name of the cross-cluster connection. You define this alias when you [set up a cross-cluster connection]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/#set-up-a-cross-cluster-connection). | `string` | Yes -`leader_index` | The index on the leader cluster that you want to replicate. | `string` | Yes -`use_roles` | The roles to use for all subsequent backend replication tasks between the indexes. Specify a `leader_cluster_role` and `follower_cluster_role`. See [Map the leader and follower cluster roles]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). | `string` | If Security plugin is enabled - -#### Sample response - -```json -{ - "acknowledged": true -} -``` - -## Stop replication -Introduced 1.1 -{: .label .label-purple } - -Terminates replication and converts the follower index to a standard index. Send this request to the follower cluster. - -#### Request - -```json -POST /_plugins/_replication//_stop -{} -``` - -#### Sample response - -```json -{ - "acknowledged": true -} -``` - -## Pause replication -Introduced 1.1 -{: .label .label-purple } - -Pauses replication of the leader index. Send this request to the follower cluster. - -#### Request - -```json -POST /_plugins/_replication//_pause -{} -``` - -You can't resume replication after it's been paused for more than 12 hours. You must [stop replication]({{site.url}}{{site.baseurl}}/replication-plugin/api/#stop-replication), delete the follower index, and restart replication of the leader. - -#### Sample response - -```json -{ - "acknowledged": true -} -``` - -## Resume replication -Introduced 1.1 -{: .label .label-purple } - -Resumes replication of the leader index. Send this request to the follower cluster. - -#### Request - -```json -POST /_plugins/_replication//_resume -{} -``` - -#### Sample response - -```json -{ - "acknowledged": true -} -``` - -## Get replication status -Introduced 1.1 -{: .label .label-purple } - -Gets the status of index replication. Possible statuses are `SYNCING`, `BOOTSTRAPING`, `PAUSED`, and `REPLICATION NOT IN PROGRESS`. Use the syncing details to measure replication lag. Send this request to the follower cluster. - -#### Request - -```json -GET /_plugins/_replication//_status -``` - -#### Sample response - -```json -{ - "status" : "SYNCING", - "reason" : "User initiated", - "leader_alias" : "my-connection-name", - "leader_index" : "leader-01", - "follower_index" : "follower-01", - "syncing_details" : { - "leader_checkpoint" : 19, - "follower_checkpoint" : 19, - "seq_no" : 0 - } -} -``` -To include shard replication details in the response, add the `&verbose=true` parameter. - -The leader and follower checkpoint values begin as negative integers and reflect the shard count (-1 for one shard, -5 for five shards, and so on). The values increment toward positive integers with each change that you make. For example, when you make a change on the leader index, the `leader_checkpoint` becomes `0`. The `follower_checkpoint` is initially still `-1` until the follower index pulls the change from the leader, at which point it increments to `0`. If the values are the same, it means the indexes are fully synced. - -## Get leader cluster stats -Introduced 1.1 -{: .label .label-purple } - -Gets information about replicated leader indexes on a specified cluster. - -#### Request - -```json -GET /_plugins/_replication/leader_stats -``` - -#### Sample response - -```json -{ - "num_replicated_indices": 2, - "operations_read": 15, - "translog_size_bytes": 1355, - "operations_read_lucene": 0, - "operations_read_translog": 15, - "total_read_time_lucene_millis": 0, - "total_read_time_translog_millis": 659, - "bytes_read": 1000, - "index_stats":{ - "leader-index-1":{ - "operations_read": 7, - "translog_size_bytes": 639, - "operations_read_lucene": 0, - "operations_read_translog": 7, - "total_read_time_lucene_millis": 0, - "total_read_time_translog_millis": 353, - "bytes_read":466 - }, - "leader-index-2":{ - "operations_read": 8, - "translog_size_bytes": 716, - "operations_read_lucene": 0, - "operations_read_translog": 8, - "total_read_time_lucene_millis": 0, - "total_read_time_translog_millis": 306, - "bytes_read": 534 - } - } -} -``` - -## Get follower cluster stats -Introduced 1.1 -{: .label .label-purple } - -Gets information about follower (syncing) indexes on a specified cluster. - -#### Request - -```json -GET /_plugins/_replication/follower_stats -``` - -#### Sample response - -```json -{ - "num_syncing_indices": 2, - "num_bootstrapping_indices": 0, - "num_paused_indices": 0, - "num_failed_indices": 0, - "num_shard_tasks": 2, - "num_index_tasks": 2, - "operations_written": 3, - "operations_read": 3, - "failed_read_requests": 0, - "throttled_read_requests": 0, - "failed_write_requests": 0, - "throttled_write_requests": 0, - "follower_checkpoint": 1, - "leader_checkpoint": 1, - "total_write_time_millis": 2290, - "index_stats":{ - "follower-index-1":{ - "operations_written": 2, - "operations_read": 2, - "failed_read_requests": 0, - "throttled_read_requests": 0, - "failed_write_requests": 0, - "throttled_write_requests": 0, - "follower_checkpoint": 1, - "leader_checkpoint": 1, - "total_write_time_millis": 1355 - }, - "follower-index-2":{ - "operations_written": 1, - "operations_read": 1, - "failed_read_requests": 0, - "throttled_read_requests": 0, - "failed_write_requests": 0, - "throttled_write_requests": 0, - "follower_checkpoint": 0, - "leader_checkpoint": 0, - "total_write_time_millis": 935 - } - } -} -``` - -## Get auto-follow stats -Introduced 1.1 -{: .label .label-purple } - -Gets information about auto-follow activity and any replication rules configured on the specified cluster. - -#### Request - -```json -GET /_plugins/_replication/autofollow_stats -``` - -#### Sample response - -```json -{ - "num_success_start_replication": 2, - "num_failed_start_replication": 0, - "num_failed_leader_calls": 0, - "failed_indices":[ - - ], - "autofollow_stats":[ - { - "name":"my-replication-rule", - "pattern":"movies*", - "num_success_start_replication": 2, - "num_failed_start_replication": 0, - "num_failed_leader_calls": 0, - "failed_indices":[ - - ] - } - ] -} -``` - -## Update settings -Introduced 1.1 -{: .label .label-purple } - -Updates settings on the follower index. - -#### Request - -```json -PUT /_plugins/_replication//_update -{ - "settings":{ - "index.number_of_shards": 4, - "index.number_of_replicas": 2 - } -} -``` - -#### Sample response - -```json -{ - "acknowledged": true -} -``` - -## Create replication rule -Introduced 1.1 -{: .label .label-purple } - -Automatically starts replication on indexes matching a specified pattern. If a new index on the leader cluster matches the pattern, OpenSearch automatically creates a follower index and begins replication. You can also use this API to update existing replication rules. - -Send this request to the follower cluster. - -Make sure to note the names of all auto-follow patterns after you create them. The replication plugin currently does not include an API operation to retrieve a list of existing patterns. -{: .tip } - -#### Request - -```json -POST /_plugins/_replication/_autofollow -{ - "leader_alias" : "", - "name": "", - "pattern": "", - "use_roles":{ - "leader_cluster_role": "", - "follower_cluster_role": "" - } -} -``` - -Specify the following options: - -Options | Description | Type | Required -:--- | :--- |:--- |:--- | -`leader_alias` | The name of the cross-cluster connection. You define this alias when you [set up a cross-cluster connection]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/#set-up-a-cross-cluster-connection). | `string` | Yes -`name` | A name for the auto-follow pattern. | `string` | Yes -`pattern` | An array of index patterns to match against indexes in the specified leader cluster. Supports wildcard characters. For example, `leader-*`. | `string` | Yes -`use_roles` | The roles to use for all subsequent backend replication tasks between the indexes. Specify a `leader_cluster_role` and `follower_cluster_role`. See [Map the leader and follower cluster roles]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). | `string` | If Security plugin is enabled - -#### Sample response - -```json -{ - "acknowledged": true -} -``` - -## Delete replication rule -Introduced 1.1 -{: .label .label-purple } - -Deletes the specified replication rule. This operation prevents any new indexes from being replicated but does not stop existing replication that the rule has already initiated. Replicated indexes remain read-only until you stop replication. - -Send this request to the follower cluster. - -#### Request - -```json -DELETE /_plugins/_replication/_autofollow -{ - "leader_alias" : "", - "name": "", -} -``` - -Specify the following options: - -Options | Description | Type | Required -:--- | :--- |:--- |:--- | -`leader_alias` | The name of the cross-cluster connection. You define this alias when you [set up a cross-cluster connection]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/#set-up-a-cross-cluster-connection). | `string` | Yes -`name` | The name of the pattern. | `string` | Yes - -#### Sample response - -```json -{ - "acknowledged": true -} -``` diff --git a/_replication-plugin/auto-follow.md b/_replication-plugin/auto-follow.md deleted file mode 100644 index ed4ecffb..00000000 --- a/_replication-plugin/auto-follow.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -layout: default -title: Auto-follow -nav_order: 20 -has_children: false - ---- - -# Auto-follow for cross-cluster replication - -Auto-follow lets you automatically replicate indexes created on the leader cluster based on matching patterns. When you create an index on the leader cluster with a name that matches a specified pattern (for example, `index-01*`), a corresponding follower index is automatically created on the follower cluster. - -You can configure multiple replication rules for a single cluster. The patterns currently only support wildcard matching. - -## Prerequisites - -You need to [set up a cross-cluster connection]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/#set-up-a-cross-cluster-connection) between two clusters before you can enable auto-follow. - -## Permissions - -If the Security plugin is enabled, make sure that non-admin users are mapped to the appropriate permissions so they can perform replication actions. For index and cluster-level permissions requirements, see [Cross-cluster replication permissions]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/). - -## Get started with auto-follow - -Replication rules are a collection of patterns that you create against a single follower cluster. When you create a replication rule, it starts by automatically replicating any *existing* indexes that match the pattern. It will then continue to replicate any *new* indexes that you create that match the pattern. - -Create a replication rule on the follower cluster: - -```bash -curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/_autofollow?pretty' -d ' -{ - "leader_alias" : "my-connection-alias", - "name": "my-replication-rule", - "pattern": "movies*", - "use_roles":{ - "leader_cluster_role": "all_access", - "follower_cluster_role": "all_access" - } -}' -``` - -If the Security plugin is disabled, you can leave out the `use_roles` parameter. If it's enabled, however, you need to specify the leader and follower cluster roles that OpenSearch uses to authenticate requests. This example uses `all_access` for simplicity, but we recommend creating a replication user on each cluster and [mapping it accordingly]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). -{: .tip } - -To test the rule, create a matching index on the leader cluster: - -```bash -curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/movies-0001?pretty' -``` - -And confirm its replica shows up on the follower cluster: - -```bash -curl -XGET -u 'admin:admin' -k 'https://localhost:9200/_cat/indices?v' -``` - -It might take several seconds for the index to appear. - -```bash -health status index uuid pri rep docs.count docs.deleted store.size pri.store.size -yellow open movies-0001 kHOxYYHxRMeszLjTD9rvSQ 1 1 0 0 208b 208b -``` - -## Retrieve replication rules - -To retrieve a list of existing replication rules that are configured on a cluster, send the following request: - -```bash -curl -XGET -u 'admin:admin' -k 'https://localhost:9200/_plugins/_replication/autofollow_stats' - -{ - "num_success_start_replication": 1, - "num_failed_start_replication": 0, - "num_failed_leader_calls": 0, - "failed_indices":[ - - ], - "autofollow_stats":[ - { - "name":"my-replication-rule", - "pattern":"movies*", - "num_success_start_replication": 1, - "num_failed_start_replication": 0, - "num_failed_leader_calls": 0, - "failed_indices":[ - - ] - } - ] -} -``` - -## Delete a replication rule - -To delete a replication rule, send the following request to the follower cluster: - -```bash -curl -XDELETE -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/_autofollow?pretty' -d ' -{ - "leader_alias" : "my-conection-alias", - "name": "my-replication-rule" -}' -``` - -When you delete a replication rule, OpenSearch stops replicating *new* indexes that match the pattern, but existing indexes that the rule previously created remain read-only and continue to replicate. If you need to stop existing replication activity and open the indexes up for writes, use the [stop replication API operation]({{site.url}}{{site.baseurl}}/replication-plugin/api/#stop-replication). \ No newline at end of file diff --git a/_replication-plugin/get-started.md b/_replication-plugin/get-started.md deleted file mode 100644 index 94ad2a3e..00000000 --- a/_replication-plugin/get-started.md +++ /dev/null @@ -1,292 +0,0 @@ ---- -layout: default -title: Get started -nav_order: 10 ---- - -# Get started with cross-cluster replication - -With cross-cluster replication, you index data to a leader index, and OpenSearch replicates that data to one or more read-only follower indexes. All subsequent operations on the leader are replicated on the follower, such as creating, updating, or deleting documents. - -## Prerequisites - -Cross-cluster replication has the following prerequisites: -- Both the leader and follower cluster must have the replication plugin installed. -- If you've overridden `node.roles` in `opensearch.yml` on the follower cluster, make sure it also includes the `remote_cluster_client` role: - - ```yaml - node.roles: [, remote_cluster_client] - ``` - -## Permissions - -Make sure the Security plugin is either enabled on both clusters or disabled on both clusters. If you disabled the Security plugin, you can skip this section. However, we strongly recommend enabling the Security plugin in production scenarios. - -If the Security plugin is enabled, make sure that non-admin users are mapped to the appropriate permissions so they can perform replication actions. For index and cluster-level permissions requirements, see [Cross-cluster replication permissions]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/). - -In addition, verify and add the distinguished names (DNs) of each follower cluster node on the leader cluster to allow connections from the followers to the leader. - -First, get the node's DN from each follower cluster: - - ```bash -curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_opendistro/_security/api/ssl/certs?pretty' - -{ - "transport_certificates_list": [ - { - "issuer_dn" : "CN=Test,OU=Server CA 1B,O=Test,C=US", - "subject_dn" : "CN=follower.test.com", # To be added under leader's nodes_dn configuration - "not_before" : "2021-11-12T00:00:00Z", - "not_after" : "2022-12-11T23:59:59Z" - } - ] -} - ``` - -Then verify that it's part of the leader cluster configuration in `opensearch.yml`. Otherwise, add it under the following setting: - - ```yaml -plugins.security.nodes_dn: - - "CN=*.leader.com, OU=SSL, O=Test, L=Test, C=DE" # Already part of the configuration - - "CN=follower.test.com" # From the above response from follower - ``` -## Example setup - -To start two single-node clusters on the same network, save this sample file as `docker-compose.yml` and run `docker-compose up`: - -```yml -version: '3' -services: - replication-node1: - image: opensearchproject/opensearch:{{site.opensearch_version}} - container_name: replication-node1 - environment: - - cluster.name=leader-cluster - - discovery.type=single-node - - bootstrap.memory_lock=true - - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - ulimits: - memlock: - soft: -1 - hard: -1 - volumes: - - opensearch-data2:/usr/share/opensearch/data - ports: - - 9201:9200 - - 9700:9600 # required for Performance Analyzer - networks: - - opensearch-net - replication-node2: - image: opensearchproject/opensearch:{{site.opensearch_version}} - container_name: replication-node2 - environment: - - cluster.name=follower-cluster - - discovery.type=single-node - - bootstrap.memory_lock=true - - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - ulimits: - memlock: - soft: -1 - hard: -1 - volumes: - - opensearch-data1:/usr/share/opensearch/data - ports: - - 9200:9200 - - 9600:9600 # required for Performance Analyzer - networks: - - opensearch-net - -volumes: - opensearch-data1: - opensearch-data2: - -networks: - opensearch-net: -``` - -After the clusters start, verify the names of each: - -```bash -curl -XGET -u 'admin:admin' -k 'https://localhost:9201' -{ - "cluster_name" : "leader-cluster", - ... -} - -curl -XGET -u 'admin:admin' -k 'https://localhost:9200' -{ - "cluster_name" : "follower-cluster", - ... -} -``` - -For this example, use port 9201 (`replication-node1`) as the leader and port 9200 (`replication-node2`) as the follower cluster. - -To get the IP address for the leader cluster, first identify its container ID: - -```bash -docker ps -CONTAINER ID IMAGE PORTS NAMES -3b8cdc698be5 opensearchproject/opensearch:{{site.opensearch_version}} 0.0.0.0:9200->9200/tcp, 0.0.0.0:9600->9600/tcp, 9300/tcp replication-node2 -731f5e8b0f4b opensearchproject/opensearch:{{site.opensearch_version}} 9300/tcp, 0.0.0.0:9201->9200/tcp, 0.0.0.0:9700->9600/tcp replication-node1 -``` - -Then get that container's IP address: - -```bash -docker inspect --format='{% raw %}{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}{% endraw %}' 731f5e8b0f4b -172.22.0.3 -``` - -## Set up a cross-cluster connection - -Cross-cluster replication follows a "pull" model, so most changes occur on the follower cluster, not the leader cluster. - -On the follower cluster, add the IP address (with port 9300) for each seed node. Because this is a single-node cluster, you only have one seed node. Provide a descriptive name for the connection, which you'll use in the request to start replication: - -```bash -curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_cluster/settings?pretty' -d ' -{ - "persistent": { - "cluster": { - "remote": { - "my-connection-alias": { - "seeds": ["172.22.0.3:9300"] - } - } - } - } -}' -``` - -## Start replication - -To get started, create an index called `leader-01` on the leader cluster: - -```bash -curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/leader-01?pretty' -``` - -Then start replication from the follower cluster. In the request body, provide the connection name and leader index that you want to replicate, along with the security roles you want to use: - -```bash -curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_start?pretty' -d ' -{ - "leader_alias": "my-connection-alias", - "leader_index": "leader-01", - "use_roles":{ - "leader_cluster_role": "all_access", - "follower_cluster_role": "all_access" - } -}' -``` - -If the Security plugin is disabled, omit the `use_roles` parameter. If it's enabled, however, you must specify the leader and follower cluster roles that OpenSearch will use to authenticate the request. This example uses `all_access` for simplicity, but we recommend creating a replication user on each cluster and [mapping it accordingly]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). -{: .tip } - -This command creates an identical read-only index named `follower-01` on the follower cluster that continuously stays updated with changes to the `leader-01` index on the leader cluster. Starting replication creates a follower index from scratch -- you can't convert an existing index to a follower index. - -## Confirm replication - -After replication starts, get the status: - -```bash -curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_status?pretty' - -{ - "status" : "SYNCING", - "reason" : "User initiated", - "leader_alias" : "my-connection-alias", - "leader_index" : "leader-01", - "follower_index" : "follower-01", - "syncing_details" : { - "leader_checkpoint" : -1, - "follower_checkpoint" : -1, - "seq_no" : 0 - } -} -``` - -Possible statuses are `SYNCING`, `BOOTSTRAPPING`, `PAUSED`, and `REPLICATION NOT IN PROGRESS`. - -The leader and follower checkpoint values begin as negative numbers and reflect the shard count (-1 for one shard, -5 for five shards, and so on). The values increment with each change and illustrate how many updates the follower is behind the leader. If the indexes are fully synced, the values are the same. - -To confirm that replication is actually happening, add a document to the leader index: - -```bash -curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/leader-01/_doc/1?pretty' -d '{"The Shining": "Stephen King"}' -``` - -Then validate the replicated content on the follower index: - -```bash -curl -XGET -k -u 'admin:admin' 'https://localhost:9200/follower-01/_search?pretty' - -{ - ... - "hits": [{ - "_index": "follower-01", - "_id": "1", - "_score": 1.0, - "_source": { - "The Shining": "Stephen King" - } - }] -} -``` - -## Pause and resume replication - -You can temporarily pause replication of an index if you need to remediate issues or reduce load on the leader cluster: - -```bash -curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_pause?pretty' -d '{}' -``` - -To confirm that replication is paused, get the status: - -```bash -curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_status?pretty' - -{ - "status" : "PAUSED", - "reason" : "User initiated", - "leader_alias" : "my-connection-alias", - "leader_index" : "leader-01", - "follower_index" : "follower-01" -} -``` - -When you're done making changes, resume replication: - -```bash -curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_resume?pretty' -d '{}' -``` - -When replication resumes, the follower index picks up any changes that were made to the leader index while replication was paused. - -Note that you can't resume replication after it's been paused for more than 12 hours. You must [stop replication]({{site.url}}{{site.baseurl}}/replication-plugin/api/#stop-replication), delete the follower index, and restart replication of the leader. - -## Stop replication - -When you no longer need to replicate an index, terminate replication from the follower cluster: - -```bash -curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_stop?pretty' -d '{}' -``` - -When you stop replication, the follower index un-follows the leader and becomes a standard index that you can write to. You can't restart replication after stopping it. - -Get the status to confirm that the index is no longer being replicated: - -```bash -curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_status?pretty' - -{ - "status" : "REPLICATION NOT IN PROGRESS" -} -``` - -You can further confirm that replication is stopped by making modifications to the leader index and confirming they don't show up on the follower index. - - diff --git a/_replication-plugin/index.md b/_replication-plugin/index.md deleted file mode 100644 index 1b643d4e..00000000 --- a/_replication-plugin/index.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -layout: default -title: Cross-cluster replication -nav_order: 1 -has_children: false -redirect_from: - - /replication-plugin/ ---- - -# Cross-cluster replication - -The cross-cluster replication plugin lets you replicate indexes, mappings, and metadata from one OpenSearch cluster to another. Cross-cluster replication has the following benefits: -- By replicating your indexes, you ensure that you can continue to handle search requests if there's an outage. -- Replicating data across geographically distant data centers minimizes the distance between the data and the application server. This reduces expensive latencies. -- You can replicate data from multiple smaller clusters to a centralized reporting cluster, which is useful when it's inefficient to query across a large network. - -Replication follows an active-passive model where the follower index (where the data is replicated) pulls data from the leader (remote) index. - -The replication plugin supports replication of indexes using wildcard pattern matching and provides commands to pause, resume, and stop replication. Once replication starts on an index, it initiates persistent background tasks on all primary shards on the follower cluster, which continuously poll corresponding shards from the leader cluster for updates. - -You can use the replication plugin with the Security plugin to encrypt cross-cluster traffic with node-to-node encryption and control access to replication activities. - -To start, see [Get started with cross-cluster replication]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/). diff --git a/_replication-plugin/permissions.md b/_replication-plugin/permissions.md deleted file mode 100644 index db7f1298..00000000 --- a/_replication-plugin/permissions.md +++ /dev/null @@ -1,78 +0,0 @@ ---- -layout: default -title: Replication security -nav_order: 30 ---- - -# Cross-cluster replication security - -You can use the [Security plugin]({{site.url}}{{site.baseurl}}/security/index/) with cross-cluster replication to limit users to certain actions. For example, you might want certain users to only perform replication activity on the leader or follower cluster. - -Because cross-cluster replication involves multiple clusters, it's possible that clusters might have different security configurations. The following configurations are supported: - -- Security plugin fully enabled on both clusters -- Security plugin enabled only for TLS on both clusters (`plugins.security.ssl_only`) -- Security plugin absent or disabled on both clusters (not recommended) - -Enable node-to-node encryption on both the leader and the follower cluster to ensure that replication traffic between the clusters is encrypted. - -## Basic permissions - -In order for non-admin users to perform replication activities, they must be mapped to the appropriate permissions. - -The Security plugin has two built-in roles that cover most replication use cases: `cross_cluster_replication_leader_full_access`, which provides replication permissions on the leader cluster, and `cross_cluster_replication_follower_full_access`, which provides replication permissions on the follower cluster. For descriptions of each, see [Predefined roles]({{site.url}}{{site.baseurl}}/security/access-control/users-roles#predefined-roles). - -If you don't want to use the default roles, you can combine individual replication [permissions]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#replication-permissions) to meet your needs. Most permissions correspond to specific REST API operations. For example, the `indices:admin/plugins/replication/index/pause` permission lets you pause replication. - -## Map the leader and follower cluster roles - -The [start replication]({{site.url}}{{site.baseurl}}/replication-plugin/api/#start-replication) and [create replication rule]({{site.url}}{{site.baseurl}}/replication-plugin/api/#create-replication-rule) operations are special cases. They involve background processes on the leader and follower clusters that must be associated with roles. When you perform one of these actions, you must explicitly pass the `leader_cluster_role` and -`follower_cluster_role` in the request, which OpenSearch then uses in all backend replication tasks. - -To enable non-admins to start replication and create replication rules, create an identical user on each cluster (for example, `replication_user`) and map them to the `cross_cluster_replication_leader_full_access` role on the remote cluster and `cross_cluster_replication_follower_full_access` on the follower cluster. For instructions, see [Map users to roles]({{site.url}}{{site.baseurl}}/security/access-control/users-roles/#map-users-to-roles). - -Then add those roles to the request, and sign it with the appropriate credentials: - -```bash -curl -XPUT -k -H 'Content-Type: application/json' -u 'replication_user:password' 'https://localhost:9200/_plugins/_replication/follower-01/_start?pretty' -d ' -{ - "leader_alias": "leader-cluster", - "leader_index": "leader-01", - "use_roles":{ - "leader_cluster_role": "cross_cluster_replication_leader_full_access", - "follower_cluster_role": "cross_cluster_replication_follower_full_access" - } -}' -``` - -You can create your own, custom leader and follower cluster roles using individual permissions, but we recommend using the default roles, which are a good fit for most use cases. - -## Replication permissions - -The following sections list the available index and cluster-level permissions for cross-cluster replication. - -### Follower cluster - -The Security plugin supports these permissions for the follower cluster: - -``` -indices:admin/plugins/replication/index/setup/validate -indices:admin/plugins/replication/index/start -indices:admin/plugins/replication/index/pause -indices:admin/plugins/replication/index/resume -indices:admin/plugins/replication/index/stop -indices:admin/plugins/replication/index/update -indices:admin/plugins/replication/index/status_check -indices:data/write/plugins/replication/changes -cluster:admin/plugins/replication/autofollow/update -``` - -### Leader cluster - -The Security plugin supports these permissions for the leader cluster: - -``` -indices:admin/plugins/replication/validate -indices:data/read/plugins/replication/file_chunk -indices:data/read/plugins/replication/changes -``` diff --git a/_replication-plugin/settings.md b/_replication-plugin/settings.md deleted file mode 100644 index 12edf865..00000000 --- a/_replication-plugin/settings.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -layout: default -title: Settings -nav_order: 40 ---- - -# Replication settings - -The replication plugin adds several settings to the standard OpenSearch cluster settings. -The settings are dynamic, so you can change the default behavior of the plugin without restarting your cluster. -You can mark settings as `persistent` or `transient`. - -For example, to update how often the follower cluster polls the leader cluster for updates: - -```json -PUT _cluster/settings -{ - "persistent": { - "plugins.replication.follower.metadata_sync_interval": "30s" - } -} -``` - -These settings manage the resources consumed by remote recoveries. We don’t recommend changing these settings; the defaults should work well for most use cases. - -Setting | Default | Description -:--- | :--- | :--- -`plugins.replication.follower.index.recovery.chunk_size` | 10 MB | The chunk size requested by the follower cluster during file transfer. Specify the chunk size as a value and unit, for example, 10 MB, 5 KB. See [Supported units]({{site.url}}{{site.baseurl}}/opensearch/units/). -`plugins.replication.follower.index.recovery.max_concurrent_file_chunks` | 4 | The number of file chunk requests that can be sent in parallel for each recovery. -`plugins.replication.follower.index.ops_batch_size` | 5000 | The number of operations that can be fetched at a time during the syncing phase of replication. -`plugins.replication.follower.concurrent_readers_per_shard` | 2 | The number of concurrent requests from the follower cluster per shard during the syncing phase of replication. -`plugins.replication.autofollow.fetch_poll_interval` | 30s | How often auto-follow tasks poll the leader cluster for new matching indexes. -`plugins.replication.follower.metadata_sync_interval` | 60s | How often the follower cluster polls the leader cluster for updated index metadata. -`plugins.replication.translog.retention_lease.pruning.enabled` | true | If enabled, prunes the translog based on retention leases on the leader index. -`plugins.replication.translog.retention_size` | 512 MB | Controls the size of the translog on the leader index. - diff --git a/_search-plugins/index.md b/_search-plugins/index.md index b366dbb0..f98b0f72 100644 --- a/_search-plugins/index.md +++ b/_search-plugins/index.md @@ -4,6 +4,7 @@ title: Search nav_order: 1 has_children: false has_toc: false +nav_exclude: true --- # Search diff --git a/_security-analytics/index.md b/_security-analytics/index.md index 156276fd..d956c9ca 100644 --- a/_security-analytics/index.md +++ b/_security-analytics/index.md @@ -4,6 +4,7 @@ title: About Security Analytics nav_order: 1 has_children: false has_toc: false +nav_exclude: true redirect_from: - /security-analytics/ --- diff --git a/_security/index.md b/_security/index.md index 48099647..f005c351 100755 --- a/_security/index.md +++ b/_security/index.md @@ -4,6 +4,7 @@ title: About Security nav_order: 1 has_children: false has_toc: false +nav_exclude: true redirect_from: - /security-plugin/ - /security-plugin/index/ diff --git a/_tools/index.md b/_tools/index.md index b669d45d..552c43c7 100644 --- a/_tools/index.md +++ b/_tools/index.md @@ -3,6 +3,7 @@ layout: default title: Tools nav_order: 50 has_children: false +nav_exclude: true redirect_from: - /clients/agents-and-ingestion-tools/ --- diff --git a/_troubleshoot/index.md b/_troubleshoot/index.md index 4322b417..f2280d04 100644 --- a/_troubleshoot/index.md +++ b/_troubleshoot/index.md @@ -3,6 +3,7 @@ layout: default title: Common issues nav_order: 1 has_toc: false +nav_exclude: true redirect_from: /troubleshoot/ --- diff --git a/_tuning-your-cluster/index.md b/_tuning-your-cluster/index.md index 125aa4c5..69d5fdbc 100644 --- a/_tuning-your-cluster/index.md +++ b/_tuning-your-cluster/index.md @@ -2,6 +2,7 @@ layout: default title: Creating a cluster nav_order: 1 +nav_exclude: true redirect_from: - /opensearch/cluster/ - /tuning-your-cluster/cluster/ diff --git a/_upgrade-to/index.md b/_upgrade-to/index.md index 5b8995ff..c6ac302f 100644 --- a/_upgrade-to/index.md +++ b/_upgrade-to/index.md @@ -2,6 +2,7 @@ layout: default title: About the migration process nav_order: 1 +nav_exclude: true redirect_from: - /upgrade-to/ ---