diff --git a/_api-reference/cluster-api/cluster-awareness.md b/_api-reference/cluster-api/cluster-awareness.md index 8e7073d9..0c0fd49d 100644 --- a/_api-reference/cluster-api/cluster-awareness.md +++ b/_api-reference/cluster-api/cluster-awareness.md @@ -4,6 +4,8 @@ title: Cluster routing and awareness nav_order: 20 parent: Cluster APIs has_children: false +redirect_from: + - /api-reference/cluster-awareness/ --- # Cluster routing and awareness diff --git a/_api-reference/cluster-api/cluster-decommission.md b/_api-reference/cluster-api/cluster-decommission.md index 6d0fcec3..e64e2675 100644 --- a/_api-reference/cluster-api/cluster-decommission.md +++ b/_api-reference/cluster-api/cluster-decommission.md @@ -4,6 +4,8 @@ title: Cluster decommission nav_order: 30 parent: Cluster APIs has_children: false +redirect_from: + - /api-reference/cluster-decommission/ --- # Cluster decommission diff --git a/_api-reference/cluster-api/cluster-health.md b/_api-reference/cluster-api/cluster-health.md index a87b796d..42475cc3 100644 --- a/_api-reference/cluster-api/cluster-health.md +++ b/_api-reference/cluster-api/cluster-health.md @@ -4,6 +4,7 @@ title: Cluster health nav_order: 40 parent: Cluster APIs has_children: false +redirect_from: /api-reference/cluster-health/ --- # Cluster health diff --git a/_api-reference/cluster-api/cluster-stats.md b/_api-reference/cluster-api/cluster-stats.md index 8097c27c..9e56b178 100644 --- a/_api-reference/cluster-api/cluster-stats.md +++ b/_api-reference/cluster-api/cluster-stats.md @@ -4,6 +4,8 @@ title: Cluster stats nav_order: 60 parent: Cluster APIs has_children: false +redirect_from: + - /api-reference/cluster-stats/ --- # Cluster stats diff --git a/_api-reference/index.md b/_api-reference/index.md index de6f13ae..8b603877 100644 --- a/_api-reference/index.md +++ b/_api-reference/index.md @@ -2,6 +2,7 @@ layout: default title: REST API reference nav_order: 1 +has_toc: true redirect_from: - /opensearch/rest-api/ --- diff --git a/_config.yml b/_config.yml index 10ba0964..c89a0728 100644 --- a/_config.yml +++ b/_config.yml @@ -8,7 +8,7 @@ permalink: /:path/ opensearch_version: 2.5.0 opensearch_dashboards_version: 2.5.0 opensearch_major_minor_version: 2.5 -lucene_version: 9_4_2 +lucene_version: 9_4_1 # Build settings markdown: kramdown @@ -37,25 +37,37 @@ collections: opensearch: permalink: /:collection/:path/ output: true + im-plugin: + permalink: /:collection/:path/ + output: true dashboards: permalink: /:collection/:path/ output: true + tuning-your-cluster: + permalink: /:collection/:path/ + output: true security: permalink: /:collection/:path/ - output: true + output: true security-analytics: permalink: /:collection/:path/ output: true search-plugins: permalink: /:collection/:path/ - output: true - im-plugin: + output: true + ml-commons-plugin: permalink: /:collection/:path/ output: true - replication-plugin: + neural-search-plugin: + permalink: /:collection/:path/ + output: true + tuning-your-cluster: permalink: /:collection/:path/ output: true - observability-plugin: + monitoring-your-cluster: + permalink: /:collection/:path/ + output: true + observing-your-data: permalink: /:collection/:path/ output: true ml-commons-plugin: @@ -64,15 +76,6 @@ collections: neural-search-plugin: permalink: /:collection/:path/ output: true - monitoring-plugins: - permalink: /:collection/:path/ - output: true - notifications-plugin: - permalink: /:collection/:path/ - output: true - job-scheduler-plugin: - permalink: /:collection/:path/ - output: true clients: permalink: /:collection/:path/ output: true @@ -106,9 +109,15 @@ just_the_docs: opensearch: name: OpenSearch nav_fold: true + im-plugin: + name: Managing Indexes + nav_fold: true dashboards: name: OpenSearch Dashboards nav_fold: true + tuning-your-cluster: + name: Tuning your cluster + nav_fold: true security: name: Security in OpenSearch nav_fold: true @@ -116,31 +125,22 @@ just_the_docs: name: Security analytics plugin nav_fold: true search-plugins: - name: Search plugins - nav_fold: true - im-plugin: - name: Index management plugin - nav_fold: true - replication-plugin: - name: Replication plugin - nav_fold: true - observability-plugin: - name: Observability plugin + name: Search nav_fold: true ml-commons-plugin: - name: ML Commons plugin + name: Machine learning nav_fold: true neural-search-plugin: - name: Neural Search plugin + name: Neural Search nav_fold: true - monitoring-plugins: - name: Monitoring plugins + tuning-your-cluster: + name: Tuning your cluster + nav_fold: true + monitoring-your-cluster: + name: Monitoring your cluster nav_fold: true - notifications-plugin: - name: Notifications plugin - nav_fold: true - job-scheduler-plugin: - name: Job Scheduler plugin + observing-your-data: + name: Observing your data nav_fold: true clients: name: Clients @@ -226,4 +226,4 @@ exclude: - vendor/gems/ - vendor/ruby/ - README.md - - .idea + - .idea \ No newline at end of file diff --git a/_data-prepper/peer-forwarder.md b/_data-prepper/peer-forwarder.md index efb62dbb..7f4ba14a 100644 --- a/_data-prepper/peer-forwarder.md +++ b/_data-prepper/peer-forwarder.md @@ -6,7 +6,7 @@ nav_order: 12 # Peer Forwarder -Peer Forwarder is an HTTP service that performs peer forwarding of an `event` between Data Prepper nodes for aggregation. This HTTP service uses a hash-ring approach to aggregate events and determine which Data Prepper node it should handle on a given trace before rerouting it to that node. Currently, Peer Forwarder is supported by the `aggregate`, `service_map_stateful`, and `otel_trace_raw` [processors]({{site.url}}{{site.baseurl}}/data-prepper/configuration/processors/processors/). +Peer Forwarder is an HTTP service that performs peer forwarding of an `event` between Data Prepper nodes for aggregation. This HTTP service uses a hash-ring approach to aggregate events and determine which Data Prepper node it should handle on a given trace before rerouting it to that node. Currently, Peer Forwarder is supported by the `aggregate`, `service_map_stateful`, and `otel_trace_raw` [processors]({{site.url}}{{site.baseurl}}/data-prepper/pipelines/configuration/processors/processors/). Peer Forwarder groups events based on the identification keys provided by the supported processors. For `service_map_stateful` and `otel_trace_raw`, the identification key is `traceId` by default and cannot be configured. The `aggregate` processor is configured using the `identification_keys` configuration option. From here, you can specify which keys to use for Peer Forwarder. See [Aggregate Processor page](https://github.com/opensearch-project/data-prepper/tree/main/data-prepper-plugins/aggregate-processor#identification_keys) for more information about identification keys. diff --git a/_monitoring-your-cluster/job-scheduler/index.md b/_monitoring-your-cluster/job-scheduler/index.md new file mode 100644 index 00000000..938d7980 --- /dev/null +++ b/_monitoring-your-cluster/job-scheduler/index.md @@ -0,0 +1,132 @@ +--- +layout: default +title: Job Scheduler +nav_order: 1 +has_children: false +has_toc: false +redirect_from: + - /job-scheduler-plugin/index/ +--- + +# Job Scheduler + +The OpenSearch Job Scheduler plugin provides a framework that can be used to build schedules for common tasks performed on your cluster. You can use Job Scheduler’s Service Provider Interface (SPI) to define schedules for cluster management tasks such as taking snapshots, managing your data’s lifecycle, and running periodic jobs. Job Scheduler has a sweeper that listens for updated events on the OpenSearch cluster and a scheduler that manages when jobs run. + +You can install the Job Scheduler plugin by following the standard [OpenSearch plugin installation]({{site.url}}{{site.baseurl}}/install-and-configure/install-opensearch/plugins/) process. The sample-extension-plugin example provided in the [Job Scheduler GitHub repository](https://github.com/opensearch-project/job-scheduler) provides a complete example of utilizing Job Scheduler when building a plugin. To define schedules, you build a plugin that implements the interfaces provided in the Job Scheduler library. You can schedule jobs by specifying an interval, or you can use a Unix cron expression such as `0 12 * * ?`, which runs at noon every day, to define a more flexible schedule. + +## Building a plugin for Job Scheduler + +OpenSearch plugin developers can extend the Job Scheduler plugin to schedule jobs to perform on the cluster. Jobs you can schedule include running aggregation queries against raw data, saving the aggregated data to a new index every hour, or continuing to monitor the shard allocation by calling the OpenSearch API and then posting the output to a webhook. + +For examples of building a plugin that uses the Job Scheduler plugin, see the Job Scheduler [README](https://github.com/opensearch-project/job-scheduler/blob/main/README.md). + +## Defining an endpoint + +You can configure your plugin's API endpoint by referencing the [example](https://github.com/opensearch-project/job-scheduler/blob/main/sample-extension-plugin/src/main/java/org/opensearch/jobscheduler/sampleextension/SampleExtensionRestHandler.java) `SampleExtensionRestHandler.java` file. Set the endpoint URL that your plugin will expose with `WATCH_INDEX_URI`: + +```java +public class SampleExtensionRestHandler extends BaseRestHandler { + public static final String WATCH_INDEX_URI = "/_plugins/scheduler_sample/watch"; +``` + +You can define the job configuration by [extending](https://github.com/opensearch-project/job-scheduler/blob/main/sample-extension-plugin/src/main/java/org/opensearch/jobscheduler/sampleextension/SampleJobParameter.java) `ScheduledJobParameter`. You can also define the fields used by your plugin, like `indexToWatch`, as shown in the [example](https://github.com/opensearch-project/job-scheduler/blob/main/sample-extension-plugin/src/main/java/org/opensearch/jobscheduler/sampleextension/SampleJobParameter.java) `SampleJobParameter` file. This job configuration will be saved as a document in an index you define, as shown in [this example](https://github.com/opensearch-project/job-scheduler/blob/main/sample-extension-plugin/src/main/java/org/opensearch/jobscheduler/sampleextension/SampleExtensionPlugin.java#L54). + +## Configuring parameters + +You can configure your plugin's parameters by referencing the [example](https://github.com/opensearch-project/job-scheduler/blob/main/sample-extension-plugin/src/main/java/org/opensearch/jobscheduler/sampleextension/SampleJobParameter.java) `SampleJobParameter.java` file and modifying it to fit your needs: + +```java +/** + * A sample job parameter. + *

+ * It adds an additional "indexToWatch" field to {@link ScheduledJobParameter}, which stores the index + * the job runner will watch. + */ +public class SampleJobParameter implements ScheduledJobParameter { + public static final String NAME_FIELD = "name"; + public static final String ENABLED_FILED = "enabled"; + public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; + public static final String LAST_UPDATE_TIME_FIELD_READABLE = "last_update_time_field"; + public static final String SCHEDULE_FIELD = "schedule"; + public static final String ENABLED_TIME_FILED = "enabled_time"; + public static final String ENABLED_TIME_FILED_READABLE = "enabled_time_field"; + public static final String INDEX_NAME_FIELD = "index_name_to_watch"; + public static final String LOCK_DURATION_SECONDS = "lock_duration_seconds"; + public static final String JITTER = "jitter"; + + private String jobName; + private Instant lastUpdateTime; + private Instant enabledTime; + private boolean isEnabled; + private Schedule schedule; + private String indexToWatch; + private Long lockDurationSeconds; + private Double jitter; +``` + +Next, configure the request parameters you would like your plugin to use with Job Scheduler. These will be based on the variables you declare when configuring your plugin. The following example shows the request parameters you set when building your plugin: + +```java +public SampleJobParameter(String id, String name, String indexToWatch, Schedule schedule, Long lockDurationSeconds, Double jitter) { + this.jobName = name; + this.indexToWatch = indexToWatch; + this.schedule = schedule; + + Instant now = Instant.now(); + this.isEnabled = true; + this.enabledTime = now; + this.lastUpdateTime = now; + this.lockDurationSeconds = lockDurationSeconds; + this.jitter = jitter; + } + + @Override + public String getName() { + return this.jobName; + } + + @Override + public Instant getLastUpdateTime() { + return this.lastUpdateTime; + } + + @Override + public Instant getEnabledTime() { + return this.enabledTime; + } + + @Override + public Schedule getSchedule() { + return this.schedule; + } + + @Override + public boolean isEnabled() { + return this.isEnabled; + } + + @Override + public Long getLockDurationSeconds() { + return this.lockDurationSeconds; + } + + @Override public Double getJitter() { + return jitter; + } +``` + +The following table describes the request parameters configured in the previous example. All the request parameters shown are required. + +| Field | Data type | Description | +:--- | :--- | :--- +| getName | String | Returns the name of the job. | +| getLastUpdateTime | Time unit | Returns the time that the job was last run. | +| getEnabledTime | Time unit | Returns the time that the job was enabled. | +| getSchedule | Unix cron | Returns the job schedule formatted in Unix cron syntax. | +| isEnabled | Boolean | Indicates whether or not the job is enabled. | +| getLockDurationSeconds | Integer | Returns the duration of time for which the job is locked. | +| getJitter | Integer | Returns the defined jitter value. | + +The logic used by your job should be defined by a class extended from `ScheduledJobRunner` in the `SampleJobParameter.java` sample file, such as `SampleJobRunner`. While the job is running, there is a locking mechanism you can use to prevent other nodes from running the same job. First, [acquire](https://github.com/opensearch-project/job-scheduler/blob/main/sample-extension-plugin/src/main/java/org/opensearch/jobscheduler/sampleextension/SampleJobRunner.java#L96) the lock. Then make sure to release the lock before the [job finishes](https://github.com/opensearch-project/job-scheduler/blob/main/sample-extension-plugin/src/main/java/org/opensearch/jobscheduler/sampleextension/SampleJobRunner.java#L116). + +For more information, see the Job Scheduler [sample extension](https://github.com/opensearch-project/job-scheduler/blob/main/sample-extension-plugin/src/main/java/org/opensearch/jobscheduler/sampleextension/SampleJobParameter.java) directory in the [Job Scheduler GitHub repo](https://github.com/opensearch-project/job-scheduler). diff --git a/_monitoring-plugins/pa/api.md b/_monitoring-your-cluster/pa/api.md similarity index 99% rename from _monitoring-plugins/pa/api.md rename to _monitoring-your-cluster/pa/api.md index f70481da..8ead2b47 100644 --- a/_monitoring-plugins/pa/api.md +++ b/_monitoring-your-cluster/pa/api.md @@ -3,6 +3,8 @@ layout: default title: API parent: Performance Analyzer nav_order: 1 +redirect_from: + - /monitoring-plugins/pa/api/ --- # Performance Analyzer API diff --git a/_monitoring-plugins/pa/dashboards.md b/_monitoring-your-cluster/pa/dashboards.md similarity index 99% rename from _monitoring-plugins/pa/dashboards.md rename to _monitoring-your-cluster/pa/dashboards.md index 561f6fa3..300095d3 100644 --- a/_monitoring-plugins/pa/dashboards.md +++ b/_monitoring-your-cluster/pa/dashboards.md @@ -3,6 +3,8 @@ layout: default title: Create PerfTop Dashboards parent: Performance Analyzer nav_order: 2 +redirect_from: + - /monitoring-plugins/pa/dashboards/ --- # PerfTop dashboards diff --git a/_monitoring-plugins/pa/index.md b/_monitoring-your-cluster/pa/index.md similarity index 99% rename from _monitoring-plugins/pa/index.md rename to _monitoring-your-cluster/pa/index.md index 45acb55c..d63b1795 100644 --- a/_monitoring-plugins/pa/index.md +++ b/_monitoring-your-cluster/pa/index.md @@ -5,6 +5,7 @@ nav_order: 58 has_children: true redirect_from: - /monitoring-plugins/pa/ + - /monitoring-plugins/pa/index/ --- # Performance analyzer diff --git a/_monitoring-plugins/pa/rca/api.md b/_monitoring-your-cluster/pa/rca/api.md similarity index 96% rename from _monitoring-plugins/pa/rca/api.md rename to _monitoring-your-cluster/pa/rca/api.md index 2d3aeb3e..cb8762cd 100644 --- a/_monitoring-plugins/pa/rca/api.md +++ b/_monitoring-your-cluster/pa/rca/api.md @@ -4,6 +4,8 @@ title: API parent: Root Cause Analysis grand_parent: Performance Analyzer nav_order: 1 +redirect_from: + - /monitoring-plugins/pa/rca/api/ --- # RCA API diff --git a/_monitoring-plugins/pa/rca/index.md b/_monitoring-your-cluster/pa/rca/index.md similarity index 95% rename from _monitoring-plugins/pa/rca/index.md rename to _monitoring-your-cluster/pa/rca/index.md index 765b9e23..cd636595 100644 --- a/_monitoring-plugins/pa/rca/index.md +++ b/_monitoring-your-cluster/pa/rca/index.md @@ -4,6 +4,8 @@ title: Root Cause Analysis nav_order: 50 parent: Performance Analyzer has_children: true +redirect_from: + - /monitoring-plugins/pa/rca/index/ --- # Root Cause Analysis diff --git a/_monitoring-plugins/pa/rca/reference.md b/_monitoring-your-cluster/pa/rca/reference.md similarity index 83% rename from _monitoring-plugins/pa/rca/reference.md rename to _monitoring-your-cluster/pa/rca/reference.md index 765942d0..2805f894 100644 --- a/_monitoring-plugins/pa/rca/reference.md +++ b/_monitoring-your-cluster/pa/rca/reference.md @@ -4,6 +4,8 @@ title: RCA Reference parent: Root Cause Analysis grand_parent: Performance Analyzer nav_order: 3 +redirect_from: + - /monitoring-plugins/pa/rca/reference/ --- # RCA reference diff --git a/_monitoring-plugins/pa/reference.md b/_monitoring-your-cluster/pa/reference.md similarity index 99% rename from _monitoring-plugins/pa/reference.md rename to _monitoring-your-cluster/pa/reference.md index caa87a8f..9fed7646 100644 --- a/_monitoring-plugins/pa/reference.md +++ b/_monitoring-your-cluster/pa/reference.md @@ -3,6 +3,8 @@ layout: default title: Metrics Reference parent: Performance Analyzer nav_order: 3 +redirect_from: + - /monitoring-plugins/pa/reference/ --- # Metrics reference diff --git a/_monitoring-plugins/ad/api.md b/_observing-your-data/ad/api.md similarity index 99% rename from _monitoring-plugins/ad/api.md rename to _observing-your-data/ad/api.md index c8149424..635914b9 100644 --- a/_monitoring-plugins/ad/api.md +++ b/_observing-your-data/ad/api.md @@ -3,6 +3,8 @@ layout: default title: Anomaly detection API parent: Anomaly detection nav_order: 1 +redirect_from: + - /monitoring-plugins/ad/api/ --- # Anomaly detection API diff --git a/_monitoring-plugins/ad/index.md b/_observing-your-data/ad/index.md similarity index 99% rename from _monitoring-plugins/ad/index.md rename to _observing-your-data/ad/index.md index 0c476878..be4fabc5 100644 --- a/_monitoring-plugins/ad/index.md +++ b/_observing-your-data/ad/index.md @@ -1,10 +1,11 @@ --- layout: default title: Anomaly detection -nav_order: 46 +nav_order: 90 has_children: true redirect_from: - /monitoring-plugins/ad/ + - /monitoring-plugins/ad/index/ --- # Anomaly detection diff --git a/_monitoring-plugins/ad/result-mapping.md b/_observing-your-data/ad/result-mapping.md similarity index 99% rename from _monitoring-plugins/ad/result-mapping.md rename to _observing-your-data/ad/result-mapping.md index 54d86705..7e1482a0 100644 --- a/_monitoring-plugins/ad/result-mapping.md +++ b/_observing-your-data/ad/result-mapping.md @@ -3,6 +3,8 @@ layout: default title: Anomaly result mapping parent: Anomaly detection nav_order: 6 +redirect_from: + - /monitoring-plugins/ad/result-mapping/ --- # Anomaly result mapping diff --git a/_monitoring-plugins/ad/security.md b/_observing-your-data/ad/security.md similarity index 98% rename from _monitoring-plugins/ad/security.md rename to _observing-your-data/ad/security.md index 777bb416..e309a410 100644 --- a/_monitoring-plugins/ad/security.md +++ b/_observing-your-data/ad/security.md @@ -4,6 +4,8 @@ title: Anomaly detection security nav_order: 10 parent: Anomaly detection has_children: false +redirect_from: + - /monitoring-plugins/ad/security/ --- # Anomaly detection security diff --git a/_monitoring-plugins/ad/settings.md b/_observing-your-data/ad/settings.md similarity index 99% rename from _monitoring-plugins/ad/settings.md rename to _observing-your-data/ad/settings.md index ca0ddf4e..66cfe6fe 100644 --- a/_monitoring-plugins/ad/settings.md +++ b/_observing-your-data/ad/settings.md @@ -3,6 +3,8 @@ layout: default title: Settings parent: Anomaly detection nav_order: 4 +redirect_from: + - /monitoring-plugins/ad/settings/ --- # Settings diff --git a/_monitoring-plugins/alerting/api.md b/_observing-your-data/alerting/api.md similarity index 99% rename from _monitoring-plugins/alerting/api.md rename to _observing-your-data/alerting/api.md index 074c64d8..811743d8 100644 --- a/_monitoring-plugins/alerting/api.md +++ b/_observing-your-data/alerting/api.md @@ -3,6 +3,8 @@ layout: default title: API parent: Alerting nav_order: 15 +redirect_from: + - /monitoring-plugins/alerting/api/ --- # Alerting API diff --git a/_monitoring-plugins/alerting/cron.md b/_observing-your-data/alerting/cron.md similarity index 97% rename from _monitoring-plugins/alerting/cron.md rename to _observing-your-data/alerting/cron.md index bba64d06..b37d13e5 100644 --- a/_monitoring-plugins/alerting/cron.md +++ b/_observing-your-data/alerting/cron.md @@ -4,6 +4,8 @@ title: Cron nav_order: 20 parent: Alerting has_children: false +redirect_from: + - /monitoring-plugins/alerting/cron/ --- # Cron expression reference diff --git a/_monitoring-plugins/alerting/index.md b/_observing-your-data/alerting/index.md similarity index 95% rename from _monitoring-plugins/alerting/index.md rename to _observing-your-data/alerting/index.md index 3495e425..fa033ce1 100644 --- a/_monitoring-plugins/alerting/index.md +++ b/_observing-your-data/alerting/index.md @@ -1,10 +1,11 @@ --- layout: default title: Alerting -nav_order: 34 +nav_order: 70 has_children: true redirect_from: - /monitoring-plugins/alerting/ + - /monitoring-plugins/alerting/index/ --- # Alerting diff --git a/_monitoring-plugins/alerting/monitors.md b/_observing-your-data/alerting/monitors.md similarity index 99% rename from _monitoring-plugins/alerting/monitors.md rename to _observing-your-data/alerting/monitors.md index 1cb92aa9..9872d3e2 100644 --- a/_monitoring-plugins/alerting/monitors.md +++ b/_observing-your-data/alerting/monitors.md @@ -4,6 +4,8 @@ title: Monitors nav_order: 1 parent: Alerting has_children: false +redirect_from: + - /monitoring-plugins/alerting/monitors/ --- # Monitors diff --git a/_monitoring-plugins/alerting/security.md b/_observing-your-data/alerting/security.md similarity index 99% rename from _monitoring-plugins/alerting/security.md rename to _observing-your-data/alerting/security.md index 0b384338..21685a64 100644 --- a/_monitoring-plugins/alerting/security.md +++ b/_observing-your-data/alerting/security.md @@ -4,6 +4,8 @@ title: Alerting security nav_order: 10 parent: Alerting has_children: false +redirect_from: + - /monitoring-plugins/alerting/security/ --- # Alerting security diff --git a/_monitoring-plugins/alerting/settings.md b/_observing-your-data/alerting/settings.md similarity index 98% rename from _monitoring-plugins/alerting/settings.md rename to _observing-your-data/alerting/settings.md index 6e44d7ae..20ef388b 100644 --- a/_monitoring-plugins/alerting/settings.md +++ b/_observing-your-data/alerting/settings.md @@ -3,6 +3,8 @@ layout: default title: Management parent: Alerting nav_order: 5 +redirect_from: + - /monitoring-plugins/alerting/settings/ --- # Management diff --git a/_observability-plugin/app-analytics.md b/_observing-your-data/app-analytics.md similarity index 95% rename from _observability-plugin/app-analytics.md rename to _observing-your-data/app-analytics.md index 03a3dfa0..44e1939a 100644 --- a/_observability-plugin/app-analytics.md +++ b/_observing-your-data/app-analytics.md @@ -1,7 +1,9 @@ --- layout: default title: Application analytics -nav_order: 80 +nav_order: 10 +redirect_from: + - /observing-your-data/app-analytics/ --- # Application analytics @@ -39,7 +41,7 @@ To see your visualizations, choose the **Panel** tab. ### Configure availability -Availability is the status of your application determined by availability levels set on a [time series metric]({{site.url}}{{site.baseurl}}/observability-plugin/app-analytics/#time-series-metric). +Availability is the status of your application determined by availability levels set on a [time series metric]({{site.url}}{{site.baseurl}}/observing-your-data/app-analytics/#time-series-metric). To create an availability level, you must configure the following: - color: The color of the availability badge on the home page. diff --git a/_observability-plugin/event-analytics.md b/_observing-your-data/event-analytics.md similarity index 95% rename from _observability-plugin/event-analytics.md rename to _observing-your-data/event-analytics.md index 030315eb..780a1f00 100644 --- a/_observability-plugin/event-analytics.md +++ b/_observing-your-data/event-analytics.md @@ -1,7 +1,9 @@ --- layout: default title: Event analytics -nav_order: 10 +nav_order: 20 +redirect_from: + - /observing-your-data/event-analytics/ --- # Event analytics @@ -28,7 +30,7 @@ For more information about building PPL queries, see [Piped Processing Language] ## Save a visualization -After Dashboards generates a visualization, you must save it if you want to return to it at a later time or if you want to add it to an [operational panel]({{site.url}}{{site.baseurl}}/observability-plugin/operational-panels). +After Dashboards generates a visualization, you must save it if you want to return to it at a later time or if you want to add it to an [operational panel]({{site.url}}{{site.baseurl}}/observing-your-data/operational-panels). To save a visualization, expand the save dropdown menu next to **Refresh**, enter a name for your visualization, then choose **Save**. You can reopen any saved visualizations on the event analytics page. diff --git a/_observability-plugin/index.md b/_observing-your-data/index.md similarity index 58% rename from _observability-plugin/index.md rename to _observing-your-data/index.md index 304cf2db..4c516855 100644 --- a/_observability-plugin/index.md +++ b/_observing-your-data/index.md @@ -1,13 +1,13 @@ --- layout: default -title: About Observability +title: Observing your data nav_order: 1 has_children: false redirect_from: - - /observability-plugin/ + - /observability-plugin/index/ --- -# About Observability +# Observing your data OpenSearch Dashboards {: .label .label-yellow :} @@ -16,12 +16,12 @@ Observability is collection of plugins and applications that let you visualize d Your experience of exploring data might differ, but if you're new to exploring data to create visualizations, we recommend trying a workflow like the following: 1. Explore data within a certain timeframe using [Piped Processing Language]({{site.url}}{{site.baseurl}}/search-plugins/sql/ppl/index). -2. Use [event analytics]({{site.url}}{{site.baseurl}}/observability-plugin/event-analytics) to turn data-driven events into visualizations. +2. Use [event analytics]({{site.url}}{{site.baseurl}}/observing-your-data/event-analytics) to turn data-driven events into visualizations. ![Sample Event Analytics View]({{site.url}}{{site.baseurl}}/images/event-analytics.png) -3. Create [operational panels]({{site.url}}{{site.baseurl}}/observability-plugin/operational-panels) and add visualizations to compare data the way you like. +3. Create [operational panels]({{site.url}}{{site.baseurl}}/observing-your-data/operational-panels) and add visualizations to compare data the way you like. ![Sample Operational Panel View]({{site.url}}{{site.baseurl}}/images/operational-panel.png) -4. Use [log analytics]({{site.url}}{{site.baseurl}}/observability-plugin/log-analytics) to transform unstructured log data. -5. Use [trace analytics]({{site.url}}{{site.baseurl}}/observability-plugin/trace/index) to create traces and dive deep into your data. +4. Use [log analytics]({{site.url}}{{site.baseurl}}/observing-your-data/log-ingestion/) to transform unstructured log data. +5. Use [trace analytics]({{site.url}}{{site.baseurl}}/observing-your-data/trace/index) to create traces and dive deep into your data. ![Sample Trace Analytics View]({{site.url}}{{site.baseurl}}/images/observability-trace.png) -6. Leverage [notebooks]({{site.url}}{{site.baseurl}}/observability-plugin/notebooks) to combine different visualizations and code blocks that you can share with team members. +6. Leverage [notebooks]({{site.url}}{{site.baseurl}}/observing-your-data/notebooks) to combine different visualizations and code blocks that you can share with team members. ![Sample Notebooks View]({{site.url}}{{site.baseurl}}/images/notebooks.png) diff --git a/_observing-your-data/log-analytics.md b/_observing-your-data/log-analytics.md new file mode 100644 index 00000000..e69de29b diff --git a/_observability-plugin/log-analytics.md b/_observing-your-data/log-ingestion.md similarity index 96% rename from _observability-plugin/log-analytics.md rename to _observing-your-data/log-ingestion.md index ad079661..c85b2ae3 100644 --- a/_observability-plugin/log-analytics.md +++ b/_observing-your-data/log-ingestion.md @@ -1,7 +1,9 @@ --- layout: default -title: Log analytics -nav_order: 70 +title: Log ingestion +nav_order: 30 +redirect_from: + - /observability-plugin/log-analytics/ --- # Log Ingestion @@ -90,4 +92,4 @@ The response should show the parsed log data: ] ``` -The same data can be viewed in OpenSearch Dashboards by visiting the **Discover** page and searching the `apache_logs` index. Remember, you must create the index in OpenSearch Dashboards if this is your first time searching for the index. +The same data can be viewed in OpenSearch Dashboards by visiting the **Discover** page and searching the `apache_logs` index. Remember, you must create the index in OpenSearch Dashboards if this is your first time searching for the index. \ No newline at end of file diff --git a/_observability-plugin/notebooks.md b/_observing-your-data/notebooks.md similarity index 98% rename from _observability-plugin/notebooks.md rename to _observing-your-data/notebooks.md index c4392696..e85750ff 100644 --- a/_observability-plugin/notebooks.md +++ b/_observing-your-data/notebooks.md @@ -2,7 +2,9 @@ layout: default title: Notebooks nav_order: 50 -redirect_from: /notebooks/ +redirect_from: + - /notebooks/ + - /observability-plugin/notebooks/ has_children: false --- diff --git a/_notifications-plugin/api.md b/_observing-your-data/notifications/api.md similarity index 99% rename from _notifications-plugin/api.md rename to _observing-your-data/notifications/api.md index 79a2180c..bc550998 100644 --- a/_notifications-plugin/api.md +++ b/_observing-your-data/notifications/api.md @@ -2,8 +2,10 @@ layout: default title: API nav_order: 50 -has_children: false +has_children: true +parent: Notifications redirect_from: + - /notifications-plugin/api/ --- # Notifications API diff --git a/_notifications-plugin/index.md b/_observing-your-data/notifications/index.md similarity index 99% rename from _notifications-plugin/index.md rename to _observing-your-data/notifications/index.md index 521d4384..c04d9787 100644 --- a/_notifications-plugin/index.md +++ b/_observing-your-data/notifications/index.md @@ -1,10 +1,11 @@ --- layout: default title: Notifications -nav_order: 1 +nav_order: 80 has_children: false redirect_from: - /notifications-plugin/ + - /notifications-plugin/index/ --- # Notifications diff --git a/_observability-plugin/observability-security.md b/_observing-your-data/observability-security.md similarity index 97% rename from _observability-plugin/observability-security.md rename to _observing-your-data/observability-security.md index aacf40c3..da3ad613 100644 --- a/_observability-plugin/observability-security.md +++ b/_observing-your-data/observability-security.md @@ -3,6 +3,8 @@ layout: default title: Observability security nav_order: 5 has_children: false +redirect_from: + - /observing-your-data/security/ --- # Observability security diff --git a/_observability-plugin/operational-panels.md b/_observing-your-data/operational-panels.md similarity index 87% rename from _observability-plugin/operational-panels.md rename to _observing-your-data/operational-panels.md index 8b8db539..1b05b9de 100644 --- a/_observability-plugin/operational-panels.md +++ b/_observing-your-data/operational-panels.md @@ -1,7 +1,9 @@ --- layout: default title: Operational panels -nav_order: 30 +nav_order: 60 +redirect_from: + - /observing-your-data/operational-panels/ --- # Operational panels @@ -16,7 +18,7 @@ If you want to start using operational panels without adding any data, expand th To create an operational panel and add visualizations: -1. From the **Add Visualization** dropdown menu, choose **Select Existing Visualization** or **Create New Visualization**, which takes you to the [event analytics]({{site.url}}{{site.baseurl}}/observability-plugin/event-analytics) explorer, where you can use PPL to create visualizations. +1. From the **Add Visualization** dropdown menu, choose **Select Existing Visualization** or **Create New Visualization**, which takes you to the [event analytics]({{site.url}}{{site.baseurl}}/observing-your-data/event-analytics) explorer, where you can use PPL to create visualizations. 1. If you're adding already existing visualizations, choose a visualization from the dropdown menu. 1. Choose **Add**. diff --git a/_observability-plugin/trace/get-started.md b/_observing-your-data/trace/getting-started.md similarity index 93% rename from _observability-plugin/trace/get-started.md rename to _observing-your-data/trace/getting-started.md index 852626a9..1e48b739 100644 --- a/_observability-plugin/trace/get-started.md +++ b/_observing-your-data/trace/getting-started.md @@ -1,11 +1,13 @@ --- layout: default -title: Get Started +title: Getting Started parent: Trace analytics nav_order: 1 +redirect_from: + - /observability-plugin/trace/get-started/ --- -# Get started with Trace Analytics +# Getting started with Trace Analytics OpenSearch Trace Analytics consists of two components---Data Prepper and the Trace Analytics OpenSearch Dashboards plugin---that fit into the OpenTelemetry and OpenSearch ecosystems. The Data Prepper repository has several [sample applications](https://github.com/opensearch-project/data-prepper/tree/main/examples) to help you get started. @@ -21,7 +23,7 @@ OpenSearch Trace Analytics consists of two components---Data Prepper and the Tra 1. [Data Prepper]({{site.url}}{{site.baseurl}}/clients/data-prepper/index/) processes the OpenTelemetry data, transforms it for use in OpenSearch, and indexes it on an OpenSearch cluster. -1. The [Trace Analytics OpenSearch Dashboards plugin]({{site.url}}{{site.baseurl}}/observability-plugin/trace/ta-dashboards/) displays the data in near real-time as a series of charts and tables, with an emphasis on service architecture, latency, error rate, and throughput. +1. The [Trace Analytics OpenSearch Dashboards plugin]({{site.url}}{{site.baseurl}}/observing-your-data/trace/ta-dashboards/) displays the data in near real-time as a series of charts and tables, with an emphasis on service architecture, latency, error rate, and throughput. ## Jaeger HotROD @@ -78,4 +80,4 @@ curl -X GET -u 'admin:admin' -k 'https://localhost:9200/otel-v1-apm-span-000001/ Navigate to `http://localhost:5601` in a web browser and choose **Trace Analytics**. You can see the results of your single click in the Jaeger HotROD web interface: the number of traces per API and HTTP method, latency trends, a color-coded map of the service architecture, and a list of trace IDs that you can use to drill down on individual operations. -If you don't see your trace, adjust the timeframe in OpenSearch Dashboards. For more information on using the plugin, see [OpenSearch Dashboards plugin]({{site.url}}{{site.baseurl}}/observability-plugin/trace/ta-dashboards/). +If you don't see your trace, adjust the timeframe in OpenSearch Dashboards. For more information on using the plugin, see [OpenSearch Dashboards plugin]({{site.url}}{{site.baseurl}}/observing-your-data/trace/ta-dashboards/). diff --git a/_observability-plugin/trace/index.md b/_observing-your-data/trace/index.md similarity index 94% rename from _observability-plugin/trace/index.md rename to _observing-your-data/trace/index.md index b2466d56..35a2ee32 100644 --- a/_observability-plugin/trace/index.md +++ b/_observing-your-data/trace/index.md @@ -1,9 +1,11 @@ --- layout: default title: Trace analytics -nav_order: 60 +nav_order: 40 has_children: true has_toc: false +redirect_from: + - /observability-plugin/trace/index/ --- # Trace analytics diff --git a/_observability-plugin/trace/ta-dashboards.md b/_observing-your-data/trace/ta-dashboards.md similarity index 95% rename from _observability-plugin/trace/ta-dashboards.md rename to _observing-your-data/trace/ta-dashboards.md index 31bb57ad..98e4f0b4 100644 --- a/_observability-plugin/trace/ta-dashboards.md +++ b/_observing-your-data/trace/ta-dashboards.md @@ -3,6 +3,8 @@ layout: default title: OpenSearch Dashboards plugin parent: Trace analytics nav_order: 50 +redirect_from: + - /observability-plugin/trace/ta-dashboards/ --- # Trace Analytics OpenSearch Dashboards plugin diff --git a/_observability-plugin/trace/trace-analytics-jaeger.md b/_observing-your-data/trace/trace-analytics-jaeger.md similarity index 99% rename from _observability-plugin/trace/trace-analytics-jaeger.md rename to _observing-your-data/trace/trace-analytics-jaeger.md index df5f63f3..f5af3c05 100644 --- a/_observability-plugin/trace/trace-analytics-jaeger.md +++ b/_observing-your-data/trace/trace-analytics-jaeger.md @@ -3,6 +3,8 @@ layout: default title: Analyzing Jaeger trace data parent: Trace analytics nav_order: 55 +redirect_from: + - /observability-plugin/trace/trace-analytics-jaeger/ --- # Analyzing Jaeger trace data diff --git a/_search-plugins/sql/ppl/functions.md b/_search-plugins/sql/ppl/functions.md index 9ae2e6cf..49a546e5 100644 --- a/_search-plugins/sql/ppl/functions.md +++ b/_search-plugins/sql/ppl/functions.md @@ -1,7 +1,7 @@ --- layout: default title: Commands -parent: PPL - Piped Processing Language +parent: PPL – Piped Processing Language grand_parent: SQL and PPL nav_order: 2 --- diff --git a/_search-plugins/sql/ppl/syntax.md b/_search-plugins/sql/ppl/syntax.md index 9ccd701f..08d6882f 100644 --- a/_search-plugins/sql/ppl/syntax.md +++ b/_search-plugins/sql/ppl/syntax.md @@ -1,7 +1,7 @@ --- layout: default title: Syntax -parent: PPL - Piped Processing Language +parent: PPL – Piped Processing Language grand_parent: SQL and PPL nav_order: 1 --- diff --git a/_security/access-control/permissions.md b/_security/access-control/permissions.md index bdfeaf39..47f2beca 100644 --- a/_security/access-control/permissions.md +++ b/_security/access-control/permissions.md @@ -3,6 +3,8 @@ layout: default title: Permissions parent: Access control nav_order: 110 +redirect_from: + - /security-plugin/access-control/permissions/ --- # Permissions diff --git a/_security/configuration/index.md b/_security/configuration/index.md index 325fe07a..81e43b13 100644 --- a/_security/configuration/index.md +++ b/_security/configuration/index.md @@ -6,6 +6,7 @@ has_children: true has_toc: false redirect_from: - /security-plugin/configuration/ + - /security-plugin/configuration/index/ --- # Security configuration diff --git a/_security/index.md b/_security/index.md index 82ae6eb2..1e8abcdb 100755 --- a/_security/index.md +++ b/_security/index.md @@ -6,6 +6,8 @@ has_children: false has_toc: false redirect_from: - /security-plugin/ + - /security-plugin/index/ + - /security/ --- # About Security in OpenSearch diff --git a/_tuning-your-cluster/availability-and-recovery/index.md b/_tuning-your-cluster/availability-and-recovery/index.md new file mode 100644 index 00000000..3a68fd4c --- /dev/null +++ b/_tuning-your-cluster/availability-and-recovery/index.md @@ -0,0 +1,9 @@ +--- +layout: default +title: Availability and Recovery +nav_order: 20 +has_children: true +has_toc: true +--- + +The following OpenSearch features help ensure consistent uptime so that your cluster can complete and scale based on your use case, as well as creating snapshots. \ No newline at end of file diff --git a/_opensearch/remote.md b/_tuning-your-cluster/availability-and-recovery/remote.md similarity index 99% rename from _opensearch/remote.md rename to _tuning-your-cluster/availability-and-recovery/remote.md index e39e4333..1d2977b8 100644 --- a/_opensearch/remote.md +++ b/_tuning-your-cluster/availability-and-recovery/remote.md @@ -1,7 +1,10 @@ --- layout: default title: Remote-backed storage -nav_order: 19 +nav_order: 40 +parent: Availability and Recovery +redirect_from: + - /opensearch/remote/ --- # Remote-backed storage diff --git a/_opensearch/search-backpressure.md b/_tuning-your-cluster/availability-and-recovery/search-backpressure.md similarity index 99% rename from _opensearch/search-backpressure.md rename to _tuning-your-cluster/availability-and-recovery/search-backpressure.md index ea45d84c..1133a7d4 100644 --- a/_opensearch/search-backpressure.md +++ b/_tuning-your-cluster/availability-and-recovery/search-backpressure.md @@ -1,8 +1,11 @@ --- layout: default title: Search backpressure -nav_order: 63 +nav_order: 60 has_children: false +parent: Availability and Recovery +redirect_from: + - /opensearch/search-backpressure/ --- # Search backpressure diff --git a/_tuning-your-cluster/availability-and-recovery/segment-replication/configuration.md b/_tuning-your-cluster/availability-and-recovery/segment-replication/configuration.md new file mode 100644 index 00000000..b336df69 --- /dev/null +++ b/_tuning-your-cluster/availability-and-recovery/segment-replication/configuration.md @@ -0,0 +1,84 @@ +--- +layout: default +title: Segment replication configuration +nav_order: 12 +parent: Segment replication +grand_parent: Availability and Recovery +--- + +# Segment replication configuration + +Segment replication is an experimental feature. Therefore, we do not recommend the use of segment replication in a production environment. For updates on the progress of segment replication or if you want to leave feedback that could help improve the feature, see the [Segment replication issue](https://github.com/opensearch-project/OpenSearch/issues/2194). +{: .warning } + +To enable the segment replication type, reference the steps below. + +## Enabling the feature flag + +There are several methods for enabling segment replication, depending on the install type. You will also need to set the replication strategy to `SEGMENT` when creating the index. + +### Enable on a node using a tarball install + +The flag is toggled using a new jvm parameter that is set either in `OPENSEARCH_JAVA_OPTS` or in config/jvm.options. + +1. Option 1: Update config/jvm.options by adding the following line: + + ````json + -Dopensearch.experimental.feature.replication_type.enabled=true + ```` + +1. Option 2: Use the `OPENSEARCH_JAVA_OPTS` environment variable: + + ````json + export OPENSEARCH_JAVA_OPTS="-Dopensearch.experimental.feature.replication_type.enabled=true" + ```` +1. Option 3: For developers using Gradle, update run.gradle by adding the following lines: + + ````json + testClusters { + runTask { + testDistribution = 'archive' + if (numZones > 1) numberOfZones = numZones + if (numNodes > 1) numberOfNodes = numNodes + systemProperty 'opensearch.experimental.feature.replication_type.enabled', 'true' + } + } + ```` + +### Enable with Docker containers + +If you're running Docker, add the following line to docker-compose.yml underneath the `opensearch-node` and `environment` section: + +````json +OPENSEARCH_JAVA_OPTS="-Dopensearch.experimental.feature.replication_type.enabled=true" # Enables segment replication +```` + +### Setting the replication strategy on the index + +To set the replication strategy to segment replication, create an index with replication.type set to `SEGMENT`: + +````json +PUT /my-index1 +{ + "settings": { + "index": { + "replication.type": "SEGMENT" + } + } +} +```` + +## Known limitations + +1. Enabling segment replication for an existing index requires [reindexing](https://github.com/opensearch-project/OpenSearch/issues/3685). +1. Rolling upgrades are currently not supported. Full cluster restarts are required when upgrading indexes using segment replication. [Issue 3881](https://github.com/opensearch-project/OpenSearch/issues/3881). +1. [Cross-cluster replication](https://github.com/opensearch-project/OpenSearch/issues/4090) does not currently use segment replication to copy between clusters. +1. Increased network congestion on primary shards. [Issue - Optimize network bandwidth on primary shards](https://github.com/opensearch-project/OpenSearch/issues/4245). +1. Shard allocation algorithms have not been updated to evenly spread primary shards across nodes. +1. Integration with remote-backed storage as the source of replication is [currently unsupported](https://github.com/opensearch-project/OpenSearch/issues/4448). + +### Further resources regarding segment replication + +1. [Known issues](https://github.com/opensearch-project/OpenSearch/issues/2194). +1. Steps for testing (link coming soon). +1. Segment replication blog post (link coming soon). \ No newline at end of file diff --git a/_tuning-your-cluster/availability-and-recovery/segment-replication/index.md b/_tuning-your-cluster/availability-and-recovery/segment-replication/index.md new file mode 100644 index 00000000..b7641f81 --- /dev/null +++ b/_tuning-your-cluster/availability-and-recovery/segment-replication/index.md @@ -0,0 +1,27 @@ +--- +layout: default +title: Segment replication +nav_order: 70 +has_children: true +parent: Availability and Recovery +redirect_from: + - /opensearch/segment-replication/ + - /opensearch/segment-replication/index/ +--- + +# Segment replication + +Segment replication is an experimental feature with OpenSearch 2.3. Therefore, we do not recommend the use of segment replication in a production environment. For updates on the progress of segment replication or if you want leave feedback that could help improve the feature, see the [Segment replication git issue](https://github.com/opensearch-project/OpenSearch/issues/2194). +{: .warning} + +With segment replication, segment files are copied across shards instead of documents being indexed on each shard copy. This improves indexing throughput and lowers resource utilization at the expense of increased network utilization. + +As an experimental feature, segment replication will be behind a feature flag and must be enabled on **each node** of a cluster and pass a new setting during index creation. +{: .note } + +### Potential use cases + +- Users who have high write loads but do not have high search requirements and are comfortable with longer refresh times. +- Users with very high loads who want to add new nodes, as you do not need to index all nodes when adding a new node to the cluster. + +This is the first step in a series of features designed to decouple reads and writes in order to lower compute costs. \ No newline at end of file diff --git a/_opensearch/shard-indexing-backpressure.md b/_tuning-your-cluster/availability-and-recovery/shard-indexing-backpressure.md similarity index 94% rename from _opensearch/shard-indexing-backpressure.md rename to _tuning-your-cluster/availability-and-recovery/shard-indexing-backpressure.md index ac58c7d3..cde2f125 100644 --- a/_opensearch/shard-indexing-backpressure.md +++ b/_tuning-your-cluster/availability-and-recovery/shard-indexing-backpressure.md @@ -3,6 +3,9 @@ layout: default title: Shard indexing backpressure nav_order: 62 has_children: true +parent: Availability and Recovery +redirect_from: + - /opensearch/shard-indexing-backpressure/ --- # Shard indexing backpressure diff --git a/_opensearch/shard-indexing-settings.md b/_tuning-your-cluster/availability-and-recovery/shard-indexing-settings.md similarity index 97% rename from _opensearch/shard-indexing-settings.md rename to _tuning-your-cluster/availability-and-recovery/shard-indexing-settings.md index 87269063..88b0ea70 100644 --- a/_opensearch/shard-indexing-settings.md +++ b/_tuning-your-cluster/availability-and-recovery/shard-indexing-settings.md @@ -2,8 +2,10 @@ layout: default title: Settings parent: Shard indexing backpressure -nav_order: 1 -has_children: false +nav_order: 50 +grand_parent: Availability and Recovery +redirect_from: + - /opensearch/shard-indexing-settings/ --- # Settings diff --git a/_opensearch/snapshots/index.md b/_tuning-your-cluster/availability-and-recovery/snapshots/index.md similarity index 75% rename from _opensearch/snapshots/index.md rename to _tuning-your-cluster/availability-and-recovery/snapshots/index.md index 192f6f02..43ceec1d 100644 --- a/_opensearch/snapshots/index.md +++ b/_tuning-your-cluster/availability-and-recovery/snapshots/index.md @@ -1,9 +1,12 @@ --- layout: default title: Snapshots -nav_order: 65 +nav_order: 30 has_children: true -redirect_from: /opensearch/snapshots/ +parent: Availability and Recovery +redirect_from: + - /opensearch/snapshots/ + - /opensearch/snapshots/index/ has_toc: false --- @@ -24,4 +27,4 @@ Snapshots have two main uses: You can take and restore snapshots using the [snapshot API]({{site.url}}{{site.baseurl}}/opensearch/snapshots/snapshot-restore). -If you need to automate taking snapshots, you can use the [Snapshot Management]({{site.url}}{{site.baseurl}}/opensearch/snapshots/snapshot-management) feature. +If you need to automate taking snapshots, you can use the [snapshot management]({{site.url}}{{site.baseurl}}/opensearch/snapshots/snapshot-management) feature. diff --git a/_opensearch/snapshots/searchable_snapshot.md b/_tuning-your-cluster/availability-and-recovery/snapshots/searchable_snapshot.md similarity index 98% rename from _opensearch/snapshots/searchable_snapshot.md rename to _tuning-your-cluster/availability-and-recovery/snapshots/searchable_snapshot.md index deab138c..f7ef3c98 100644 --- a/_opensearch/snapshots/searchable_snapshot.md +++ b/_tuning-your-cluster/availability-and-recovery/snapshots/searchable_snapshot.md @@ -3,7 +3,9 @@ layout: default title: Searchable snapshots parent: Snapshots nav_order: 40 -has_children: false +grand_parent: Availability and Recovery +redirect_from: + - /opensearch/snapshots/searchable_snapshot/ --- # Searchable snapshots diff --git a/_opensearch/snapshots/sm-api.md b/_tuning-your-cluster/availability-and-recovery/snapshots/sm-api.md similarity index 97% rename from _opensearch/snapshots/sm-api.md rename to _tuning-your-cluster/availability-and-recovery/snapshots/sm-api.md index 70b779ef..b5013807 100644 --- a/_opensearch/snapshots/sm-api.md +++ b/_tuning-your-cluster/availability-and-recovery/snapshots/sm-api.md @@ -1,14 +1,17 @@ --- layout: default -title: Snapshot Management API +title: Snapshot management API parent: Snapshots nav_order: 30 has_children: false +grand_parent: Availability and Recovery +redirect_from: + - /opensearch/snapshots/sm-api/ --- # Snapshot Management API -Use the [Snapshot Management (SM)]({{site.url}}{{site.baseurl}}/opensearch/snapshots/snapshot-restore#take-snapshots) API to automate [taking snapshots]({{site.url}}{{site.baseurl}}/opensearch/snapshots/snapshot-restore#take-snapshots). +Use the snapshot management (SM) API to automate [taking snapshots]({{site.url}}{{site.baseurl}}/opensearch/snapshots/snapshot-restore#take-snapshots). --- @@ -182,7 +185,7 @@ Parameter | Type | Description `snapshot_config.indices` | String | The names of the indexes in the snapshot. Multiple index names are separated by `,`. Supports wildcards (`*`). Optional. Default is `*` (all indexes). `snapshot_config.repository` | String | The repository in which to store snapshots. Required. `snapshot_config.ignore_unavailable` | Boolean | Do you want to ignore unavailable indexes? Optional. Default is `false`. -`snapshot_config.include_global_state` | Boolean | Do you want to include cluster state? Optional. Default is `true` because of [Security plugin considerations]({{site.url}}{{site.baseurl}}/opensearch/snapshots/snapshot-restore/#security-plugin-considerations). +`snapshot_config.include_global_state` | Boolean | Do you want to include cluster state? Optional. Default is `true` because of [Security plugin considerations]({{site.url}}{{site.baseurl}}/tuning-your-cluster/availability-and-recovery/snapshots/snapshot-restore#security-considerations). `snapshot_config.partial` | Boolean | Do you want to allow partial snapshots? Optional. Default is `false`. `snapshot_config.metadata` | Object | Metadata in the form of key/value pairs. Optional. `creation` | Object | Configuration for snapshot creation. Required. diff --git a/_opensearch/snapshots/snapshot-management.md b/_tuning-your-cluster/availability-and-recovery/snapshots/snapshot-management.md similarity index 94% rename from _opensearch/snapshots/snapshot-management.md rename to _tuning-your-cluster/availability-and-recovery/snapshots/snapshot-management.md index 8cedd0b4..9a25b286 100644 --- a/_opensearch/snapshots/snapshot-management.md +++ b/_tuning-your-cluster/availability-and-recovery/snapshots/snapshot-management.md @@ -1,14 +1,17 @@ --- layout: default -title: Snapshot Management +title: Snapshot management parent: Snapshots nav_order: 20 has_children: false +grand_parent: Availability and Recovery +redirect_from: + - /opensearch/snapshots/snapshot-management/ --- -# Snapshot Management +# Snapshot management -Snapshot Management (SM) lets you automate [taking snapshots]({{site.url}}{{site.baseurl}}/opensearch/snapshots/snapshot-restore#take-snapshots). To use this feature, you need to install the [Index Management (IM) Plugin]({{site.url}}{{site.baseurl}}/im-plugin). Snapshots store only incremental changes since the last snapshot. Thus, while taking an initial snapshot may be a heavy operation, subsequent snapshots have minimal overhead. To set up automatic snapshots, you have to create an SM policy with a desired SM schedule and configuration. +Snapshot management (SM) lets you automate [taking snapshots]({{site.url}}{{site.baseurl}}/opensearch/snapshots/snapshot-restore#take-snapshots). To use this feature, you need to install the [Index Management (IM) Plugin]({{site.url}}{{site.baseurl}}/im-plugin). Snapshots store only incremental changes since the last snapshot. Thus, while taking an initial snapshot may be a heavy operation, subsequent snapshots have minimal overhead. To set up automatic snapshots, you have to create an SM policy with a desired SM schedule and configuration. When you create an SM policy, its document ID is given the name `-sm-policy`. Because of this, SM policies have to obey the following rules: @@ -18,7 +21,7 @@ When you create an SM policy, its document ID is given the name `-s SM-created snapshots have names in the format `--`. Two snapshots created by different policies at the same time always have different names because of the `` prefix. To avoid name collisions within the same policy, each snapshot's name contains a random string suffix. -Each policy has associated metadata that stores the policy status. Snapshot Management saves SM policies and metadata in the system index and reads them from the system index. Thus, Snapshot Management depends on the OpenSearch cluster's indexing and searching functions. The policy's metadata keeps information about the latest creation and deletion only. The metadata is read before running every scheduled job so that SM can continue execution from the previous job's state. You can view the metadata using the [explain API]({{site.url}}{{site.baseurl}}/opensearch/snapshots/sm-api#explain). +Each policy has associated metadata that stores the policy status. Snapshot management saves SM policies and metadata in the system index and reads them from the system index. Thus, Snapshot Management depends on the OpenSearch cluster's indexing and searching functions. The policy's metadata keeps information about the latest creation and deletion only. The metadata is read before running every scheduled job so that SM can continue execution from the previous job's state. You can view the metadata using the [explain API]({{site.url}}{{site.baseurl}}/opensearch/snapshots/sm-api#explain). An SM schedule is a custom [cron]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/cron) expression. It consists of two parts: a creation schedule and a deletion schedule. You must set up a creation schedule that specifies the frequency and timing of snapshot creation. Optionally, you can set up a separate schedule for deleting snapshots. @@ -44,7 +47,7 @@ We don't recommend setting up the same repository for multiple SM policies with ## Failure management -If a snapshot operation fails, it is retried a maximum of three times. The failure message is saved in `metadata.latest_execution` and is overwritten when a subsequent snapshot operation starts. You can view the failure message using the [explain API]({{site.url}}{{site.baseurl}}/opensearch/snapshots/sm-api#explain). When using OpenSearch Dashboards, you can view the failure message on the [policy details page]({{site.url}}{{site.baseurl}}/dashboards/admin-ui-index/sm-dashboards/#enable-disable-or-delete-sm-policies). Possible reasons for failure include red index status and shard reallocation. +If a snapshot operation fails, it is retried a maximum of three times. The failure message is saved in `metadata.latest_execution` and is overwritten when a subsequent snapshot operation starts. You can view the failure message using the [explain API]({{site.url}}{{site.baseurl}}/opensearch/snapshots/sm-api#explain). When using OpenSearch Dashboards, you can view the failure message on the [policy details page]({{site.url}}{{site.baseurl}}/dashboards/admin-ui-index/sm-dashboards#view-edit-or-delete-an-sm-policy). Possible reasons for failure include red index status and shard reallocation. ## Security diff --git a/_opensearch/snapshots/snapshot-restore.md b/_tuning-your-cluster/availability-and-recovery/snapshots/snapshot-restore.md similarity index 99% rename from _opensearch/snapshots/snapshot-restore.md rename to _tuning-your-cluster/availability-and-recovery/snapshots/snapshot-restore.md index bfb7a302..8d0d5909 100644 --- a/_opensearch/snapshots/snapshot-restore.md +++ b/_tuning-your-cluster/availability-and-recovery/snapshots/snapshot-restore.md @@ -4,6 +4,9 @@ title: Take and restore snapshots parent: Snapshots nav_order: 10 has_children: false +grand_parent: Availability and Recovery +redirect_from: + - /opensearch/snapshots/snapshot-restore/ --- # Take and restore snapshots diff --git a/_opensearch/stats-api.md b/_tuning-your-cluster/availability-and-recovery/stats-api.md similarity index 99% rename from _opensearch/stats-api.md rename to _tuning-your-cluster/availability-and-recovery/stats-api.md index 050d0bb6..4ac5e4e6 100644 --- a/_opensearch/stats-api.md +++ b/_tuning-your-cluster/availability-and-recovery/stats-api.md @@ -3,7 +3,10 @@ layout: default title: Stats API parent: Shard indexing backpressure nav_order: 2 +grand_parent: Availability and Recovery has_children: false +redirect_from: + - /opensearch/stats-api/ --- # Stats API diff --git a/_tuning-your-cluster/cluster-manager-task-throttling.md b/_tuning-your-cluster/cluster-manager-task-throttling.md new file mode 100644 index 00000000..0ccb5149 --- /dev/null +++ b/_tuning-your-cluster/cluster-manager-task-throttling.md @@ -0,0 +1,107 @@ +--- +layout: default +title: Cluster manager task throttling +nav_order: 10 +has_children: false +--- + +# Cluster manager task throttling + +For many cluster state updates, such as defining a mapping or creating an index, nodes submit tasks to the cluster manager. The cluster manager maintains a pending task queue for these tasks and runs them in a single-threaded environment. When nodes send tens of thousands of resource-intensive tasks, like `put-mapping` or snapshot tasks, these tasks can pile up in the queue and flood the cluster manager. This affects the cluster manager's performance and may in turn affect the availability of the whole cluster. + +The first line of defense is to implement mechanisms in the caller nodes to avoid task overload on the cluster manager. However, even with those mechanisms in place, the cluster manager needs a built-in way to protect itself: cluster manager task throttling. + +To turn on cluster manager task throttling, you need to set throttling limits. The cluster manager uses the throttling limits to determine whether to reject a task. + +The cluster manager rejects a task based on its type. For any incoming task, the cluster manager evaluates the total number of tasks of the same type in the pending task queue. If this number exceeds the threshold for this task type, the cluster manager rejects the incoming task. Rejecting a task does not affect tasks of a different type. For example, if the cluster manager rejects a `put-mapping` task, it can still accept a subsequent `create-index` task. + +When the cluster manager rejects a task, the node performs retries with exponential backoff to resubmit the task to the cluster manager. If retries are unsuccessful within the timeout period, OpenSearch returns a cluster timeout error. + +## Setting throttling limits + +You can set throttling limits by specifying them in the `cluster_manager.throttling.thresholds` object and updating the [OpenSearch cluster settings]({{site.url}}{{site.baseurl}}/api-reference/cluster-settings). The setting is dynamic, so you can change the behavior of this feature without restarting your cluster. + +By default, throttling is disabled for all task types. +{: .note} + +The request has the following format: + +```json +PUT _cluster/settings +{ + "persistent": { + "cluster_manager.throttling.thresholds" : { + "" : { + "value" : + } + } + } +} +``` + +The following table describes the `cluster_manager.throttling.thresholds` object. + +Field Name | Description +:--- | :--- +task-type | The task type. See [supported task types](#supported-task-types) for a list of valid values. +value | The maximum number of tasks of the `task-type` type in the cluster manager's pending task queue. Default is `-1` (no task throttling). + +## Supported task types + +The following task types are supported: + +- `create-index` +- `update-settings` +- `cluster-update-settings` +- `auto-create` +- `delete-index` +- `delete-dangling-index` +- `create-data-stream` +- `remove-data-stream` +- `rollover-index` +- `index-aliases` +- `put-mapping` +- `create-index-template` +- `remove-index-template` +- `create-component-template` +- `remove-component-template` +- `create-index-template-v2` +- `remove-index-template-v2` +- `put-pipeline` +- `delete-pipeline` +- `create-persistent-task` +- `finish-persistent-task` +- `remove-persistent-task` +- `update-task-state` +- `put-script` +- `delete-script` +- `put-repository` +- `delete-repository` +- `create-snapshot` +- `delete-snapshot` +- `update-snapshot-state` +- `restore-snapshot` +- `cluster-reroute-api` + +#### Sample request + +The following request sets the throttling threshold for the `put-mapping` task type to 100: + +```json +PUT _cluster/settings +{ + "persistent": { + "cluster_manager.throttling.thresholds": { + "put-mapping": { + "value": 100 + } + } + } +} +``` + +Set the threshold to `-1` to disable throttling for a task type. +{: .note} + + + diff --git a/_opensearch/cluster.md b/_tuning-your-cluster/cluster.md similarity index 99% rename from _opensearch/cluster.md rename to _tuning-your-cluster/cluster.md index dc4ecfa1..0bd08c1f 100644 --- a/_opensearch/cluster.md +++ b/_tuning-your-cluster/cluster.md @@ -1,10 +1,12 @@ --- layout: default -title: Cluster formation -nav_order: 7 +title: Creating a cluster +nav_order: 8 +redirect_from: + - /opensearch/cluster/ --- -# Cluster formation +# Creating a cluster Before diving into OpenSearch and searching and aggregating data, you first need to create an OpenSearch cluster. diff --git a/_tuning-your-cluster/replication-plugin/api.md b/_tuning-your-cluster/replication-plugin/api.md new file mode 100644 index 00000000..60a0a375 --- /dev/null +++ b/_tuning-your-cluster/replication-plugin/api.md @@ -0,0 +1,394 @@ +--- +layout: default +title: API +nav_order: 50 +parent: Cross-cluster replication +redirect_from: + - /replication-plugin/api/ +--- + +# Cross-cluster replication API + +Use these replication operations to programmatically manage cross-cluster replication. + +#### Table of contents +- TOC +{:toc} + +## Start replication +Introduced 1.1 +{: .label .label-purple } + +Initiate replication of an index from the leader cluster to the follower cluster. Send this request to the follower cluster. + + +#### Request + +```json +PUT /_plugins/_replication//_start +{ + "leader_alias":"", + "leader_index":"", + "use_roles":{ + "leader_cluster_role":"", + "follower_cluster_role":"" + } +} +``` + +Specify the following options: + +Options | Description | Type | Required +:--- | :--- |:--- |:--- | +`leader_alias` | The name of the cross-cluster connection. You define this alias when you [set up a cross-cluster connection]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/#set-up-a-cross-cluster-connection). | `string` | Yes +`leader_index` | The index on the leader cluster that you want to replicate. | `string` | Yes +`use_roles` | The roles to use for all subsequent backend replication tasks between the indexes. Specify a `leader_cluster_role` and `follower_cluster_role`. See [Map the leader and follower cluster roles]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). | `string` | If security plugin is enabled + +#### Sample response + +```json +{ + "acknowledged": true +} +``` + +## Stop replication +Introduced 1.1 +{: .label .label-purple } + +Terminates replication and converts the follower index to a standard index. Send this request to the follower cluster. + +#### Request + +```json +POST /_plugins/_replication//_stop +{} +``` + +#### Sample response + +```json +{ + "acknowledged": true +} +``` + +## Pause replication +Introduced 1.1 +{: .label .label-purple } + +Pauses replication of the leader index. Send this request to the follower cluster. + +#### Request + +```json +POST /_plugins/_replication//_pause +{} +``` + +You can't resume replication after it's been paused for more than 12 hours. You must [stop replication]({{site.url}}{{site.baseurl}}/replication-plugin/api/#stop-replication), delete the follower index, and restart replication of the leader. + +#### Sample response + +```json +{ + "acknowledged": true +} +``` + +## Resume replication +Introduced 1.1 +{: .label .label-purple } + +Resumes replication of the leader index. Send this request to the follower cluster. + +#### Request + +```json +POST /_plugins/_replication//_resume +{} +``` + +#### Sample response + +```json +{ + "acknowledged": true +} +``` + +## Get replication status +Introduced 1.1 +{: .label .label-purple } + +Gets the status of index replication. Possible statuses are `SYNCING`, `BOOTSTRAPING`, `PAUSED`, and `REPLICATION NOT IN PROGRESS`. Use the syncing details to measure replication lag. Send this request to the follower cluster. + +#### Request + +```json +GET /_plugins/_replication//_status +``` + +#### Sample response + +```json +{ + "status" : "SYNCING", + "reason" : "User initiated", + "leader_alias" : "my-connection-name", + "leader_index" : "leader-01", + "follower_index" : "follower-01", + "syncing_details" : { + "leader_checkpoint" : 19, + "follower_checkpoint" : 19, + "seq_no" : 0 + } +} +``` +To include shard replication details in the response, add the `&verbose=true` parameter. + +The leader and follower checkpoint values begin as negative integers and reflect the shard count (-1 for one shard, -5 for five shards, and so on). The values increment toward positive integers with each change that you make. For example, when you make a change on the leader index, the `leader_checkpoint` becomes `0`. The `follower_checkpoint` is initially still `-1` until the follower index pulls the change from the leader, at which point it increments to `0`. If the values are the same, it means the indexes are fully synced. + +## Get leader cluster stats +Introduced 1.1 +{: .label .label-purple } + +Gets information about replicated leader indexes on a specified cluster. + +#### Request + +```json +GET /_plugins/_replication/leader_stats +``` + +#### Sample response + +```json +{ + "num_replicated_indices": 2, + "operations_read": 15, + "translog_size_bytes": 1355, + "operations_read_lucene": 0, + "operations_read_translog": 15, + "total_read_time_lucene_millis": 0, + "total_read_time_translog_millis": 659, + "bytes_read": 1000, + "index_stats":{ + "leader-index-1":{ + "operations_read": 7, + "translog_size_bytes": 639, + "operations_read_lucene": 0, + "operations_read_translog": 7, + "total_read_time_lucene_millis": 0, + "total_read_time_translog_millis": 353, + "bytes_read":466 + }, + "leader-index-2":{ + "operations_read": 8, + "translog_size_bytes": 716, + "operations_read_lucene": 0, + "operations_read_translog": 8, + "total_read_time_lucene_millis": 0, + "total_read_time_translog_millis": 306, + "bytes_read": 534 + } + } +} +``` + +## Get follower cluster stats +Introduced 1.1 +{: .label .label-purple } + +Gets information about follower (syncing) indexes on a specified cluster. + +#### Request + +```json +GET /_plugins/_replication/follower_stats +``` + +#### Sample response + +```json +{ + "num_syncing_indices": 2, + "num_bootstrapping_indices": 0, + "num_paused_indices": 0, + "num_failed_indices": 0, + "num_shard_tasks": 2, + "num_index_tasks": 2, + "operations_written": 3, + "operations_read": 3, + "failed_read_requests": 0, + "throttled_read_requests": 0, + "failed_write_requests": 0, + "throttled_write_requests": 0, + "follower_checkpoint": 1, + "leader_checkpoint": 1, + "total_write_time_millis": 2290, + "index_stats":{ + "follower-index-1":{ + "operations_written": 2, + "operations_read": 2, + "failed_read_requests": 0, + "throttled_read_requests": 0, + "failed_write_requests": 0, + "throttled_write_requests": 0, + "follower_checkpoint": 1, + "leader_checkpoint": 1, + "total_write_time_millis": 1355 + }, + "follower-index-2":{ + "operations_written": 1, + "operations_read": 1, + "failed_read_requests": 0, + "throttled_read_requests": 0, + "failed_write_requests": 0, + "throttled_write_requests": 0, + "follower_checkpoint": 0, + "leader_checkpoint": 0, + "total_write_time_millis": 935 + } + } +} +``` + +## Get auto-follow stats +Introduced 1.1 +{: .label .label-purple } + +Gets information about auto-follow activity and any replication rules configured on the specified cluster. + +#### Request + +```json +GET /_plugins/_replication/autofollow_stats +``` + +#### Sample response + +```json +{ + "num_success_start_replication": 2, + "num_failed_start_replication": 0, + "num_failed_leader_calls": 0, + "failed_indices":[ + + ], + "autofollow_stats":[ + { + "name":"my-replication-rule", + "pattern":"movies*", + "num_success_start_replication": 2, + "num_failed_start_replication": 0, + "num_failed_leader_calls": 0, + "failed_indices":[ + + ] + } + ] +} +``` + +## Update settings +Introduced 1.1 +{: .label .label-purple } + +Updates settings on the follower index. + +#### Request + +```json +PUT /_plugins/_replication//_update +{ + "settings":{ + "index.number_of_shards": 4, + "index.number_of_replicas": 2 + } +} +``` + +#### Sample response + +```json +{ + "acknowledged": true +} +``` + +## Create replication rule +Introduced 1.1 +{: .label .label-purple } + +Automatically starts replication on indexes matching a specified pattern. If a new index on the leader cluster matches the pattern, OpenSearch automatically creates a follower index and begins replication. You can also use this API to update existing replication rules. + +Send this request to the follower cluster. + +Make sure to note the names of all auto-follow patterns after you create them. The replication plugin currently does not include an API operation to retrieve a list of existing patterns. +{: .tip } + +#### Request + +```json +POST /_plugins/_replication/_autofollow +{ + "leader_alias" : "", + "name": "", + "pattern": "", + "use_roles":{ + "leader_cluster_role": "", + "follower_cluster_role": "" + } +} +``` + +Specify the following options: + +Options | Description | Type | Required +:--- | :--- |:--- |:--- | +`leader_alias` | The name of the cross-cluster connection. You define this alias when you [set up a cross-cluster connection]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/#set-up-a-cross-cluster-connection). | `string` | Yes +`name` | A name for the auto-follow pattern. | `string` | Yes +`pattern` | An array of index patterns to match against indexes in the specified leader cluster. Supports wildcard characters. For example, `leader-*`. | `string` | Yes +`use_roles` | The roles to use for all subsequent backend replication tasks between the indexes. Specify a `leader_cluster_role` and `follower_cluster_role`. See [Map the leader and follower cluster roles]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). | `string` | If security plugin is enabled + +#### Sample response + +```json +{ + "acknowledged": true +} +``` + +## Delete replication rule +Introduced 1.1 +{: .label .label-purple } + +Deletes the specified replication rule. This operation prevents any new indexes from being replicated but does not stop existing replication that the rule has already initiated. Replicated indexes remain read-only until you stop replication. + +Send this request to the follower cluster. + +#### Request + +```json +DELETE /_plugins/_replication/_autofollow +{ + "leader_alias" : "", + "name": "", +} +``` + +Specify the following options: + +Options | Description | Type | Required +:--- | :--- |:--- |:--- | +`leader_alias` | The name of the cross-cluster connection. You define this alias when you [set up a cross-cluster connection]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/#set-up-a-cross-cluster-connection). | `string` | Yes +`name` | The name of the pattern. | `string` | Yes + +#### Sample response + +```json +{ + "acknowledged": true +} +``` diff --git a/_tuning-your-cluster/replication-plugin/auto-follow.md b/_tuning-your-cluster/replication-plugin/auto-follow.md new file mode 100644 index 00000000..d3103df9 --- /dev/null +++ b/_tuning-your-cluster/replication-plugin/auto-follow.md @@ -0,0 +1,106 @@ +--- +layout: default +title: Auto-follow +nav_order: 20 +parent: Cross-cluster replication +redirect_from: + - /replication-plugin/auto-follow/ +--- + +# Auto-follow for cross-cluster replication + +Auto-follow lets you automatically replicate indexes created on the leader cluster based on matching patterns. When you create an index on the leader cluster with a name that matches a specified pattern (for example, `index-01*`), a corresponding follower index is automatically created on the follower cluster. + +You can configure multiple replication rules for a single cluster. The patterns currently only support wildcard matching. + +## Prerequisites + +You need to [set up a cross-cluster connection]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/#set-up-a-cross-cluster-connection) between two clusters before you can enable auto-follow. + +## Permissions + +If the security plugin is enabled, make sure that non-admin users are mapped to the appropriate permissions so they can perform replication actions. For index and cluster-level permissions requirements, see [Cross-cluster replication permissions]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/). + +## Get started with auto-follow + +Replication rules are a collection of patterns that you create against a single follower cluster. When you create a replication rule, it starts by automatically replicating any *existing* indexes that match the pattern. It will then continue to replicate any *new* indexes that you create that match the pattern. + +Create a replication rule on the follower cluster: + +```bash +curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/_autofollow?pretty' -d ' +{ + "leader_alias" : "my-connection-alias", + "name": "my-replication-rule", + "pattern": "movies*", + "use_roles":{ + "leader_cluster_role": "all_access", + "follower_cluster_role": "all_access" + } +}' +``` + +If the security plugin is disabled, you can leave out the `use_roles` parameter. If it's enabled, however, you need to specify the leader and follower cluster roles that OpenSearch uses to authenticate requests. This example uses `all_access` for simplicity, but we recommend creating a replication user on each cluster and [mapping it accordingly]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). +{: .tip } + +To test the rule, create a matching index on the leader cluster: + +```bash +curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/movies-0001?pretty' +``` + +And confirm its replica shows up on the follower cluster: + +```bash +curl -XGET -u 'admin:admin' -k 'https://localhost:9200/_cat/indices?v' +``` + +It might take several seconds for the index to appear. + +```bash +health status index uuid pri rep docs.count docs.deleted store.size pri.store.size +yellow open movies-0001 kHOxYYHxRMeszLjTD9rvSQ 1 1 0 0 208b 208b +``` + +## Retrieve replication rules + +To retrieve a list of existing replication rules that are configured on a cluster, send the following request: + +```bash +curl -XGET -u 'admin:admin' -k 'https://localhost:9200/_plugins/_replication/autofollow_stats' + +{ + "num_success_start_replication": 1, + "num_failed_start_replication": 0, + "num_failed_leader_calls": 0, + "failed_indices":[ + + ], + "autofollow_stats":[ + { + "name":"my-replication-rule", + "pattern":"movies*", + "num_success_start_replication": 1, + "num_failed_start_replication": 0, + "num_failed_leader_calls": 0, + "failed_indices":[ + + ] + } + ] +} +``` + +## Delete a replication rule + +To delete a replication rule, send the following request to the follower cluster: + +```bash +curl -XDELETE -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/_autofollow?pretty' -d ' +{ + "leader_alias" : "my-conection-alias", + "name": "my-replication-rule" +}' +``` + +When you delete a replication rule, OpenSearch stops replicating *new* indexes that match the pattern, but existing indexes that the rule previously created remain read-only and continue to replicate. If you need to stop existing replication activity and open the indexes up for writes, use the [stop replication API operation]({{site.url}}{{site.baseurl}}/replication-plugin/api/#stop-replication). \ No newline at end of file diff --git a/_tuning-your-cluster/replication-plugin/getting-started.md b/_tuning-your-cluster/replication-plugin/getting-started.md new file mode 100644 index 00000000..05f515f3 --- /dev/null +++ b/_tuning-your-cluster/replication-plugin/getting-started.md @@ -0,0 +1,295 @@ +--- +layout: default +title: Getting started +nav_order: 15 +parent: Cross-cluster replication +redirect_from: + - /replication-plugin/get-started/ +--- + +# Getting started with cross-cluster replication + +With cross-cluster replication, you index data to a leader index, and OpenSearch replicates that data to one or more read-only follower indexes. All subsequent operations on the leader are replicated on the follower, such as creating, updating, or deleting documents. + +## Prerequisites + +Cross-cluster replication has the following prerequisites: +- Both the leader and follower cluster must have the replication plugin installed. +- If you've overridden `node.roles` in `opensearch.yml` on the follower cluster, make sure it also includes the `remote_cluster_client` role: + + ```yaml + node.roles: [, remote_cluster_client] + ``` + +## Permissions + +Make sure the security plugin is either enabled on both clusters or disabled on both clusters. If you disabled the security plugin, you can skip this section. However, we strongly recommend enabling the security plugin in production scenarios. + +If the security plugin is enabled, make sure that non-admin users are mapped to the appropriate permissions so they can perform replication actions. For index and cluster-level permissions requirements, see [Cross-cluster replication permissions]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/). + +In addition, verify and add the distinguished names (DNs) of each follower cluster node on the leader cluster to allow connections from the followers to the leader. + +First, get the node's DN from each follower cluster: + + ```bash +curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_opendistro/_security/api/ssl/certs?pretty' + +{ + "transport_certificates_list": [ + { + "issuer_dn" : "CN=Test,OU=Server CA 1B,O=Test,C=US", + "subject_dn" : "CN=follower.test.com", # To be added under leader's nodes_dn configuration + "not_before" : "2021-11-12T00:00:00Z", + "not_after" : "2022-12-11T23:59:59Z" + } + ] +} + ``` + +Then verify that it's part of the leader cluster configuration in `opensearch.yml`. Otherwise, add it under the following setting: + + ```yaml +plugins.security.nodes_dn: + - "CN=*.leader.com, OU=SSL, O=Test, L=Test, C=DE" # Already part of the configuration + - "CN=follower.test.com" # From the above response from follower + ``` +## Example setup + +To start two single-node clusters on the same network, save this sample file as `docker-compose.yml` and run `docker-compose up`: + +```yml +version: '3' +services: + replication-node1: + image: opensearchproject/opensearch:{{site.opensearch_version}} + container_name: replication-node1 + environment: + - cluster.name=leader-cluster + - discovery.type=single-node + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - opensearch-data2:/usr/share/opensearch/data + ports: + - 9201:9200 + - 9700:9600 # required for Performance Analyzer + networks: + - opensearch-net + replication-node2: + image: opensearchproject/opensearch:{{site.opensearch_version}} + container_name: replication-node2 + environment: + - cluster.name=follower-cluster + - discovery.type=single-node + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - opensearch-data1:/usr/share/opensearch/data + ports: + - 9200:9200 + - 9600:9600 # required for Performance Analyzer + networks: + - opensearch-net + +volumes: + opensearch-data1: + opensearch-data2: + +networks: + opensearch-net: +``` + +After the clusters start, verify the names of each: + +```bash +curl -XGET -u 'admin:admin' -k 'https://localhost:9201' +{ + "cluster_name" : "leader-cluster", + ... +} + +curl -XGET -u 'admin:admin' -k 'https://localhost:9200' +{ + "cluster_name" : "follower-cluster", + ... +} +``` + +For this example, use port 9201 (`replication-node1`) as the leader and port 9200 (`replication-node2`) as the follower cluster. + +To get the IP address for the leader cluster, first identify its container ID: + +```bash +docker ps +CONTAINER ID IMAGE PORTS NAMES +3b8cdc698be5 opensearchproject/opensearch:{{site.opensearch_version}} 0.0.0.0:9200->9200/tcp, 0.0.0.0:9600->9600/tcp, 9300/tcp replication-node2 +731f5e8b0f4b opensearchproject/opensearch:{{site.opensearch_version}} 9300/tcp, 0.0.0.0:9201->9200/tcp, 0.0.0.0:9700->9600/tcp replication-node1 +``` + +Then get that container's IP address: + +```bash +docker inspect --format='{% raw %}{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}{% endraw %}' 731f5e8b0f4b +172.22.0.3 +``` + +## Set up a cross-cluster connection + +Cross-cluster replication follows a "pull" model, so most changes occur on the follower cluster, not the leader cluster. + +On the follower cluster, add the IP address (with port 9300) for each seed node. Because this is a single-node cluster, you only have one seed node. Provide a descriptive name for the connection, which you'll use in the request to start replication: + +```bash +curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_cluster/settings?pretty' -d ' +{ + "persistent": { + "cluster": { + "remote": { + "my-connection-alias": { + "seeds": ["172.22.0.3:9300"] + } + } + } + } +}' +``` + +## Start replication + +To get started, create an index called `leader-01` on the leader cluster: + +```bash +curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/leader-01?pretty' +``` + +Then start replication from the follower cluster. In the request body, provide the connection name and leader index that you want to replicate, along with the security roles you want to use: + +```bash +curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_start?pretty' -d ' +{ + "leader_alias": "my-connection-alias", + "leader_index": "leader-01", + "use_roles":{ + "leader_cluster_role": "all_access", + "follower_cluster_role": "all_access" + } +}' +``` + +If the security plugin is disabled, omit the `use_roles` parameter. If it's enabled, however, you must specify the leader and follower cluster roles that OpenSearch will use to authenticate the request. This example uses `all_access` for simplicity, but we recommend creating a replication user on each cluster and [mapping it accordingly]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). +{: .tip } + +This command creates an identical read-only index named `follower-01` on the follower cluster that continuously stays updated with changes to the `leader-01` index on the leader cluster. Starting replication creates a follower index from scratch -- you can't convert an existing index to a follower index. + +## Confirm replication + +After replication starts, get the status: + +```bash +curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_status?pretty' + +{ + "status" : "SYNCING", + "reason" : "User initiated", + "leader_alias" : "my-connection-alias", + "leader_index" : "leader-01", + "follower_index" : "follower-01", + "syncing_details" : { + "leader_checkpoint" : -1, + "follower_checkpoint" : -1, + "seq_no" : 0 + } +} +``` + +Possible statuses are `SYNCING`, `BOOTSTRAPPING`, `PAUSED`, and `REPLICATION NOT IN PROGRESS`. + +The leader and follower checkpoint values begin as negative numbers and reflect the shard count (-1 for one shard, -5 for five shards, and so on). The values increment with each change and illustrate how many updates the follower is behind the leader. If the indexes are fully synced, the values are the same. + +To confirm that replication is actually happening, add a document to the leader index: + +```bash +curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/leader-01/_doc/1?pretty' -d '{"The Shining": "Stephen King"}' +``` + +Then validate the replicated content on the follower index: + +```bash +curl -XGET -k -u 'admin:admin' 'https://localhost:9200/follower-01/_search?pretty' + +{ + ... + "hits": [{ + "_index": "follower-01", + "_id": "1", + "_score": 1.0, + "_source": { + "The Shining": "Stephen King" + } + }] +} +``` + +## Pause and resume replication + +You can temporarily pause replication of an index if you need to remediate issues or reduce load on the leader cluster: + +```bash +curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_pause?pretty' -d '{}' +``` + +To confirm that replication is paused, get the status: + +```bash +curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_status?pretty' + +{ + "status" : "PAUSED", + "reason" : "User initiated", + "leader_alias" : "my-connection-alias", + "leader_index" : "leader-01", + "follower_index" : "follower-01" +} +``` + +When you're done making changes, resume replication: + +```bash +curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_resume?pretty' -d '{}' +``` + +When replication resumes, the follower index picks up any changes that were made to the leader index while replication was paused. + +Note that you can't resume replication after it's been paused for more than 12 hours. You must [stop replication]({{site.url}}{{site.baseurl}}/replication-plugin/api/#stop-replication), delete the follower index, and restart replication of the leader. + +## Stop replication + +When you no longer need to replicate an index, terminate replication from the follower cluster: + +```bash +curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_stop?pretty' -d '{}' +``` + +When you stop replication, the follower index un-follows the leader and becomes a standard index that you can write to. You can't restart replication after stopping it. + +Get the status to confirm that the index is no longer being replicated: + +```bash +curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_status?pretty' + +{ + "status" : "REPLICATION NOT IN PROGRESS" +} +``` + +You can further confirm that replication is stopped by making modifications to the leader index and confirming they don't show up on the follower index. + + diff --git a/_tuning-your-cluster/replication-plugin/index.md b/_tuning-your-cluster/replication-plugin/index.md new file mode 100644 index 00000000..46eede78 --- /dev/null +++ b/_tuning-your-cluster/replication-plugin/index.md @@ -0,0 +1,24 @@ +--- +layout: default +title: Cross-cluster replication +nav_order: 12 +has_children: true +redirect_from: + - /replication-plugin/ + - /replication-plugin/index/ +--- + +# Cross-cluster replication + +The cross-cluster replication plugin lets you replicate indexes, mappings, and metadata from one OpenSearch cluster to another. Cross-cluster replication has the following benefits: +- By replicating your indexes, you ensure that you can continue to handle search requests if there's an outage. +- Replicating data across geographically distant data centers minimizes the distance between the data and the application server. This reduces expensive latencies. +- You can replicate data from multiple smaller clusters to a centralized reporting cluster, which is useful when it's inefficient to query across a large network. + +Replication follows an active-passive model where the follower index (where the data is replicated) pulls data from the leader (remote) index. + +The replication plugin supports replication of indexes using wildcard pattern matching and provides commands to pause, resume, and stop replication. Once replication starts on an index, it initiates persistent background tasks on all primary shards on the follower cluster, which continuously poll corresponding shards from the leader cluster for updates. + +You can use the replication plugin with the security plugin to encrypt cross-cluster traffic with node-to-node encryption and control access to replication activities. + +To start, see [Get started with cross-cluster replication]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/). diff --git a/_tuning-your-cluster/replication-plugin/permissions.md b/_tuning-your-cluster/replication-plugin/permissions.md new file mode 100644 index 00000000..9c8a8b10 --- /dev/null +++ b/_tuning-your-cluster/replication-plugin/permissions.md @@ -0,0 +1,81 @@ +--- +layout: default +title: Replication security +nav_order: 30 +parent: Cross-cluster replication +redirect_from: + - /replication-plugin/permissions/ +--- + +# Cross-cluster replication security + +You can use the [security plugin]({{site.url}}{{site.baseurl}}/security/index/) with cross-cluster replication to limit users to certain actions. For example, you might want certain users to only perform replication activity on the leader or follower cluster. + +Because cross-cluster replication involves multiple clusters, it's possible that clusters might have different security configurations. The following configurations are supported: + +- Security plugin fully enabled on both clusters +- Security plugin enabled only for TLS on both clusters (`plugins.security.ssl_only`) +- Security plugin absent or disabled on both clusters (not recommended) + +Enable node-to-node encryption on both the leader and the follower cluster to ensure that replication traffic between the clusters is encrypted. + +## Basic permissions + +In order for non-admin users to perform replication activities, they must be mapped to the appropriate permissions. + +The security plugin has two built-in roles that cover most replication use cases: `cross_cluster_replication_leader_full_access`, which provides replication permissions on the leader cluster, and `cross_cluster_replication_follower_full_access`, which provides replication permissions on the follower cluster. For descriptions of each, see [Predefined roles]({{site.url}}{{site.baseurl}}/security/access-control/users-roles#predefined-roles). + +If you don't want to use the default roles, you can combine individual replication [permissions]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#replication-permissions) to meet your needs. Most permissions correspond to specific REST API operations. For example, the `indices:admin/plugins/replication/index/pause` permission lets you pause replication. + +## Map the leader and follower cluster roles + +The [start replication]({{site.url}}{{site.baseurl}}/replication-plugin/api/#start-replication) and [create replication rule]({{site.url}}{{site.baseurl}}/replication-plugin/api/#create-replication-rule) operations are special cases. They involve background processes on the leader and follower clusters that must be associated with roles. When you perform one of these actions, you must explicitly pass the `leader_cluster_role` and +`follower_cluster_role` in the request, which OpenSearch then uses in all backend replication tasks. + +To enable non-admins to start replication and create replication rules, create an identical user on each cluster (for example, `replication_user`) and map them to the `cross_cluster_replication_leader_full_access` role on the remote cluster and `cross_cluster_replication_follower_full_access` on the follower cluster. For instructions, see [Map users to roles]({{site.url}}{{site.baseurl}}/security/access-control/users-roles/#map-users-to-roles). + +Then add those roles to the request, and sign it with the appropriate credentials: + +```bash +curl -XPUT -k -H 'Content-Type: application/json' -u 'replication_user:password' 'https://localhost:9200/_plugins/_replication/follower-01/_start?pretty' -d ' +{ + "leader_alias": "leader-cluster", + "leader_index": "leader-01", + "use_roles":{ + "leader_cluster_role": "cross_cluster_replication_leader_full_access", + "follower_cluster_role": "cross_cluster_replication_follower_full_access" + } +}' +``` + +You can create your own, custom leader and follower cluster roles using individual permissions, but we recommend using the default roles, which are a good fit for most use cases. + +## Replication permissions + +The following sections list the available index and cluster-level permissions for cross-cluster replication. + +### Follower cluster + +The security plugin supports these permissions for the follower cluster: + +``` +indices:admin/plugins/replication/index/setup/validate +indices:admin/plugins/replication/index/start +indices:admin/plugins/replication/index/pause +indices:admin/plugins/replication/index/resume +indices:admin/plugins/replication/index/stop +indices:admin/plugins/replication/index/update +indices:admin/plugins/replication/index/status_check +indices:data/write/plugins/replication/changes +cluster:admin/plugins/replication/autofollow/update +``` + +### Leader cluster + +The security plugin supports these permissions for the leader cluster: + +``` +indices:admin/plugins/replication/validate +indices:data/read/plugins/replication/file_chunk +indices:data/read/plugins/replication/changes +``` diff --git a/_tuning-your-cluster/replication-plugin/settings.md b/_tuning-your-cluster/replication-plugin/settings.md new file mode 100644 index 00000000..4a5d266d --- /dev/null +++ b/_tuning-your-cluster/replication-plugin/settings.md @@ -0,0 +1,39 @@ +--- +layout: default +title: Replication settings +nav_order: 40 +parent: Cross-cluster replication +redirect_from: + - /replication-plugin/settings/ +--- + +# Replication settings + +The replication plugin adds several settings to the standard OpenSearch cluster settings. +The settings are dynamic, so you can change the default behavior of the plugin without restarting your cluster. +You can mark settings as `persistent` or `transient`. + +For example, to update how often the follower cluster polls the leader cluster for updates: + +```json +PUT _cluster/settings +{ + "persistent": { + "plugins.replication.follower.metadata_sync_interval": "30s" + } +} +``` + +These settings manage the resources consumed by remote recoveries. We don’t recommend changing these settings; the defaults should work well for most use cases. + +Setting | Default | Description +:--- | :--- | :--- +`plugins.replication.follower.index.recovery.chunk_size` | 10 MB | The chunk size requested by the follower cluster during file transfer. Specify the chunk size as a value and unit, for example, 10 MB, 5 KB. See [Supported units]({{site.url}}{{site.baseurl}}/opensearch/units/). +`plugins.replication.follower.index.recovery.max_concurrent_file_chunks` | 4 | The number of file chunk requests that can be sent in parallel for each recovery. +`plugins.replication.follower.index.ops_batch_size` | 5000 | The number of operations that can be fetched at a time during the syncing phase of replication. +`plugins.replication.follower.concurrent_readers_per_shard` | 2 | The number of concurrent requests from the follower cluster per shard during the syncing phase of replication. +`plugins.replication.autofollow.fetch_poll_interval` | 30s | How often auto-follow tasks poll the leader cluster for new matching indexes. +`plugins.replication.follower.metadata_sync_interval` | 60s | How often the follower cluster polls the leader cluster for updated index metadata. +`plugins.replication.translog.retention_lease.pruning.enabled` | true | If enabled, prunes the translog based on retention leases on the leader index. +`plugins.replication.translog.retention_size` | 512 MB | Controls the size of the translog on the leader index. + diff --git a/about.md b/about.md index 1b6a45fd..13842071 100644 --- a/about.md +++ b/about.md @@ -17,7 +17,7 @@ OpenSearch is a distributed search and analytics engine based on [Apache Lucene] Unsurprisingly, people often use search engines like OpenSearch as the backend for a search application---think [Wikipedia](https://en.wikipedia.org/wiki/Wikipedia:FAQ/Technical#What_software_is_used_to_run_Wikipedia?) or an online store. It offers excellent performance and can scale up and down as the needs of the application grow or shrink. -An equally popular, but less obvious use case is log analytics, in which you take the logs from an application, feed them into OpenSearch, and use the rich search and visualization functionality to identify issues. For example, a malfunctioning web server might throw a 500 error 0.5% of the time, which can be hard to notice unless you have a real-time graph of all HTTP status codes that the server has thrown in the past four hours. You can use [OpenSearch Dashboards]({{site.url}}{{site.baseurl}}/dashboards/) to build these sorts of visualizations from data in OpenSearch. +An equally popular, but less obvious use case is log analytics, in which you take the logs from an application, feed them into OpenSearch, and use the rich search and visualization functionality to identify issues. For example, a malfunctioning web server might throw a 500 error 0.5% of the time, which can be hard to notice unless you have a real-time graph of all HTTP status codes that the server has thrown in the past four hours. You can use [OpenSearch Dashboards]({{site.url}}{{site.baseurl}}/dashboards/index/) to build these sorts of visualizations from data in OpenSearch. ## Clusters and nodes