Merge branch 'main' into index-apis
This commit is contained in:
Keith Chan
GitHub
commit
eecaeab15f
|
|
@ -0,0 +1,2 @@
|
|||
# This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
|
||||
* @opensearch-project/docs
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
name: Developer Certificate of Origin Check
|
||||
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Get PR Commits
|
||||
id: 'get-pr-commits'
|
||||
uses: tim-actions/get-pr-commits@v1.1.0
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: DCO Check
|
||||
uses: tim-actions/dco@v1.1.0
|
||||
with:
|
||||
commits: ${{ steps.get-pr-commits.outputs.commits }}
|
||||
|
|
@ -26,7 +26,6 @@ Community contributions remain essential in keeping this documentation comprehen
|
|||
|
||||
If you encounter problems or have questions when contributing to the documentation, these people can help:
|
||||
|
||||
- [aetter](https://github.com/aetter)
|
||||
- [ashwinkumar12345](https://github.com/ashwinkumar12345)
|
||||
- [keithhc2](https://github.com/keithhc2)
|
||||
- [snyder114](https://github.com/snyder114)
|
||||
|
|
@ -196,17 +195,17 @@ If you're making major changes to the documentation and need to see the rendered
|
|||
## New releases
|
||||
|
||||
1. Branch.
|
||||
1. Change the `opensearch_version` and `opensearch_major_minor_version` variables in `_config.yml`.
|
||||
1. Change the `opensearch_version`, `opensearch_major_minor_version`, and `lucene_version` variables in `_config.yml`.
|
||||
1. Start up a new cluster using the updated Docker Compose file in `docs/install/docker.md`.
|
||||
1. Update the version table in `version-history.md`.
|
||||
|
||||
Use `curl -XGET https://localhost:9200 -u admin:admin -k` to verify the OpenSearch version.
|
||||
Use `curl -XGET https://localhost:9200 -u admin:admin -k` to verify the OpenSearch and Lucene versions.
|
||||
|
||||
1. Update the plugin compatibility table in `docs/install/plugin.md`.
|
||||
1. Update the plugin compatibility table in `_opensearch/install/plugin.md`.
|
||||
|
||||
Use `curl -XGET https://localhost:9200/_cat/plugins -u admin:admin -k` to get the correct version strings.
|
||||
|
||||
1. Update the plugin compatibility table in `docs/opensearch-dashboards/plugins.md`.
|
||||
1. Update the plugin compatibility table in `_dashboards/install/plugins.md`.
|
||||
|
||||
Use `docker ps` to find the ID for the OpenSearch Dashboards node. Then use `docker exec -it <opensearch-dashboards-node-id> /bin/bash` to get shell access. Finally, run `./bin/opensearch-dashboards-plugin list` to get the plugins and version strings.
|
||||
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ Some users report compatibility issues with ingest pipelines on these versions o
|
|||
|
||||
### Compatibility Matrix for Logstash
|
||||
|
||||
| | Logstash OSS 7.x to 7.11.x | Logstash OSS 7.12.x\* | Logstash 7.13.x without OpenSearch output plugin | Logstash 7.13.x with OpenSearch output plugin |
|
||||
| | Logstash OSS 7.x to 7.11.x | Logstash OSS 7.12.x\* | Logstash 7.13.x-7.16.x without OpenSearch output plugin | Logstash 7.13.x-7.16.x with OpenSearch output plugin |
|
||||
| :---| :--- | :--- | :--- | :--- |
|
||||
| Elasticsearch OSS 7.x to 7.9.x | *Yes* | *Yes* | *No* | *Yes* |
|
||||
| Elasticsearch OSS 7.10.2 | *Yes* | *Yes* | *No* | *Yes* |
|
||||
|
|
|
|||
|
|
@ -0,0 +1,145 @@
|
|||
---
|
||||
layout: default
|
||||
title: Go client
|
||||
nav_order: 80
|
||||
---
|
||||
|
||||
# Go client
|
||||
|
||||
The OpenSearch Go client lets you connect your Go application with the data in your OpenSearch cluster.
|
||||
|
||||
|
||||
## Setup
|
||||
|
||||
If you're creating a new project:
|
||||
|
||||
```go
|
||||
go mod init
|
||||
```
|
||||
|
||||
To add the client to your project, import it like any other module:
|
||||
|
||||
```go
|
||||
go get github.com/opensearch-project/opensearch-go
|
||||
```
|
||||
|
||||
## Sample code
|
||||
|
||||
This sample code creates a client, adds an index with non-default settings, inserts a document, searches for the document, deletes the document, and finally deletes the index:
|
||||
|
||||
```go
|
||||
package main
|
||||
import (
|
||||
"os"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
opensearch "github.com/opensearch-project/opensearch-go"
|
||||
opensearchapi "github.com/opensearch-project/opensearch-go/opensearchapi"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
const IndexName = "go-test-index1"
|
||||
func main() {
|
||||
// Initialize the client with SSL/TLS enabled.
|
||||
client, err := opensearch.NewClient(opensearch.Config{
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||||
},
|
||||
Addresses: []string{"https://localhost:9200"},
|
||||
Username: "admin", // For testing only. Don't store credentials in code.
|
||||
Password: "admin",
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println("cannot initialize", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Print OpenSearch version information on console.
|
||||
fmt.Println(client.Info())
|
||||
|
||||
// Define index mapping.
|
||||
mapping := strings.NewReader(`{
|
||||
'settings': {
|
||||
'index': {
|
||||
'number_of_shards': 4
|
||||
}
|
||||
}
|
||||
}`)
|
||||
|
||||
// Create an index with non-default settings.
|
||||
res := opensearchapi.IndicesCreateRequest{
|
||||
Index: IndexName,
|
||||
Body: mapping,
|
||||
}
|
||||
fmt.Println("creating index", res)
|
||||
|
||||
// Add a document to the index.
|
||||
document := strings.NewReader(`{
|
||||
"title": "Moneyball",
|
||||
"director": "Bennett Miller",
|
||||
"year": "2011"
|
||||
}`)
|
||||
|
||||
docId := "1"
|
||||
req := opensearchapi.IndexRequest{
|
||||
Index: IndexName,
|
||||
DocumentID: docId,
|
||||
Body: document,
|
||||
}
|
||||
insertResponse, err := req.Do(context.Background(), client)
|
||||
if err != nil {
|
||||
fmt.Println("failed to insert document ", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println(insertResponse)
|
||||
|
||||
// Search for the document.
|
||||
content := strings.NewReader(`{
|
||||
"size": 5,
|
||||
"query": {
|
||||
"multi_match": {
|
||||
"query": "miller",
|
||||
"fields": ["title^2", "director"]
|
||||
}
|
||||
}
|
||||
}`)
|
||||
|
||||
search := opensearchapi.SearchRequest{
|
||||
Body: content,
|
||||
}
|
||||
|
||||
searchResponse, err := search.Do(context.Background(), client)
|
||||
if err != nil {
|
||||
fmt.Println("failed to search document ", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println(searchResponse)
|
||||
|
||||
// Delete the document.
|
||||
delete := opensearchapi.DeleteRequest{
|
||||
Index: IndexName,
|
||||
DocumentID: docId,
|
||||
}
|
||||
|
||||
deleteResponse, err := delete.Do(context.Background(), client)
|
||||
if err != nil {
|
||||
fmt.Println("failed to delete document ", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("deleting document")
|
||||
fmt.Println(deleteResponse)
|
||||
|
||||
// Delete previously created index.
|
||||
deleteIndex := opensearchapi.IndicesDeleteRequest{
|
||||
Index: []string{IndexName},
|
||||
}
|
||||
|
||||
deleteIndexResponse, err := deleteIndex.Do(context.Background(), client)
|
||||
if err != nil {
|
||||
fmt.Println("failed to delete index ", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("deleting index", deleteIndexResponse)
|
||||
}
|
||||
```
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
---
|
||||
layout: default
|
||||
title: Grafana
|
||||
nav_order: 150
|
||||
has_children: false
|
||||
---
|
||||
|
||||
# Grafana support
|
||||
|
||||
Grafana has a data source plugin that lets you explore and visualize your OpenSearch data. For information on getting started with the plugin, see the [Grafana overview page](https://grafana.com/grafana/plugins/grafana-opensearch-datasource/).
|
||||
|
|
@ -9,6 +9,21 @@ redirect_from:
|
|||
|
||||
# OpenSearch client compatibility
|
||||
|
||||
OpenSearch provides clients for several popular programming languages, with more coming. In general, clients are compatible with clusters running the same major version of OpenSearch (`major.minor.patch`).
|
||||
|
||||
For example, a 1.0.0 client works with an OpenSearch 1.1.0 cluster, but might not support any non-breaking API changes in OpenSearch 1.1.0. A 1.2.0 client works with the same cluster, but might allow you to pass unsupported options in certain functions. We recommend using the same version for both, but if your tests pass after a cluster upgrade, you don't necessarily need to upgrade your clients immediately.
|
||||
|
||||
{% comment %}
|
||||
* [OpenSearch Java client]({{site.url}}{{site.baseurl}}/clients/java/)
|
||||
{% endcomment %}
|
||||
* [OpenSearch Python client]({{site.url}}{{site.baseurl}}/clients/python/)
|
||||
* [OpenSearch JavaScript (Node.js) client]({{site.url}}{{site.baseurl}}/clients/javascript/)
|
||||
* [OpenSearch Go client]({{site.url}}{{site.baseurl}}/clients/go/)
|
||||
* [OpenSearch PHP client]({{site.url}}{{site.baseurl}}/clients/php/)
|
||||
|
||||
|
||||
## Legacy clients
|
||||
|
||||
Most clients that work with Elasticsearch OSS 7.10.2 *should* work with OpenSearch, but the latest versions of those clients might include license or version checks that artificially break compatibility. This page includes recommendations around which versions of those clients to use for best compatibility with OpenSearch.
|
||||
|
||||
Client | Recommended version
|
||||
|
|
@ -17,8 +32,9 @@ Client | Recommended version
|
|||
[Java high-level REST client](https://search.maven.org/artifact/org.elasticsearch.client/elasticsearch-rest-high-level-client/7.13.4/jar) | 7.13.4
|
||||
[Python Elasticsearch client](https://pypi.org/project/elasticsearch/7.13.4/) | 7.13.4
|
||||
[Elasticsearch Node.js client](https://www.npmjs.com/package/@elastic/elasticsearch/v/7.13.0) | 7.13.0
|
||||
[Ruby Elasticsearch client](https://rubygems.org/gems/elasticsearch/versions/7.13.0) | 7.13.0
|
||||
|
||||
Clients exist for a wide variety of languages, so if you test a client and verify that it works, please [submit a PR](https://github.com/opensearch-project/documentation-website/pulls) and add it to this table.
|
||||
If you test a legacy client and verify that it works, please [submit a PR](https://github.com/opensearch-project/documentation-website/pulls) and add it to this table.
|
||||
|
||||
|
||||
{% comment %}
|
||||
|
|
|
|||
|
|
@ -1,31 +1,45 @@
|
|||
---
|
||||
layout: default
|
||||
title: Java high-level REST client
|
||||
nav_order: 97
|
||||
nav_order: 60
|
||||
---
|
||||
|
||||
# Java high-level REST client
|
||||
|
||||
The Elasticsearch OSS Java high-level REST client allows you to interact with your OpenSearch clusters and indices through Java methods and data structures rather than HTTP methods and JSON.
|
||||
|
||||
You submit requests to your cluster using request objects, which allows you to create indices, add data to documents, or complete other operations with your cluster. In return, you get back response objects that have all of the available information, such as the associated index or ID, from your cluster.
|
||||
The OpenSearch Java high-level REST client lets you interact with your OpenSearch clusters and indices through Java methods and data structures rather than HTTP methods and JSON.
|
||||
|
||||
## Setup
|
||||
|
||||
To start using the Elasticsearch OSS Java high-level REST client, ensure that you have the following dependency in your project's `pom.xml` file:
|
||||
To start using the OpenSearch Java high-level REST client, ensure that you have the following dependency in your project's `pom.xml` file:
|
||||
|
||||
```
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>elasticsearch-rest-high-level-client</artifactId>
|
||||
<version>7.10.2</version>
|
||||
<groupId>org.opensearch.client</groupId>
|
||||
<artifactId>opensearch-rest-high-level-client</artifactId>
|
||||
<version>{{site.opensearch_version}}</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
You can now start your OpenSearch cluster. The 7.10.2 high-level REST client works with the 1.x versions of OpenSearch.
|
||||
You can now start your OpenSearch cluster. The OpenSearch 1.x high-level REST client works with the 1.x versions of OpenSearch.
|
||||
|
||||
## Security
|
||||
|
||||
Before using the REST client in your Java application, you must configure the application's truststore to connect to the security plugin. If you are using self-signed certificates or demo configurations, you can use the following command to create a custom truststore and add in root authority certificates.
|
||||
|
||||
If you're using certificates from a trusted Certificate Authority (CA), you don't need to configure the truststore.
|
||||
|
||||
```bash
|
||||
keytool -import <path-to-cert> -alias <alias-to-call-cert> -keystore <truststore-name>
|
||||
```
|
||||
|
||||
You can now point your Java client to the truststore and set basic authentication credentials that can access a secure cluster (refer to the sample code below on how to do so).
|
||||
|
||||
If you run into issues when configuring security, see [common issues]({{site.url}}{{site.baseurl}}/troubleshoot/index) and [troubleshoot TLS]({{site.url}}{{site.baseurl}}/troubleshoot/tls).
|
||||
|
||||
## Sample code
|
||||
|
||||
This code example uses basic credentials that come with the default OpenSearch configuration. If you’re using the OpenSearch Java high-level REST client with your own OpenSearch cluster, be sure to change the code to use your own credentials.
|
||||
|
||||
```java
|
||||
import org.apache.http.HttpHost;
|
||||
import org.apache.http.auth.AuthScope;
|
||||
|
|
@ -33,22 +47,21 @@ import org.apache.http.auth.UsernamePasswordCredentials;
|
|||
import org.apache.http.client.CredentialsProvider;
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider;
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
|
||||
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
|
||||
import org.elasticsearch.action.delete.DeleteRequest;
|
||||
import org.elasticsearch.action.delete.DeleteResponse;
|
||||
import org.elasticsearch.action.get.GetRequest;
|
||||
import org.elasticsearch.action.get.GetResponse;
|
||||
import org.elasticsearch.action.index.IndexRequest;
|
||||
import org.elasticsearch.action.index.IndexResponse;
|
||||
import org.elasticsearch.action.support.master.AcknowledgedResponse;
|
||||
import org.elasticsearch.client.RequestOptions;
|
||||
import org.elasticsearch.client.RestClient;
|
||||
import org.elasticsearch.client.RestClientBuilder;
|
||||
import org.elasticsearch.client.RestHighLevelClient;
|
||||
import org.elasticsearch.client.indices.CreateIndexRequest;
|
||||
import org.elasticsearch.client.indices.CreateIndexResponse;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.xcontent.XContentType;
|
||||
import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
|
||||
import org.opensearch.action.delete.DeleteRequest;
|
||||
import org.opensearch.action.delete.DeleteResponse;
|
||||
import org.opensearch.action.get.GetRequest;
|
||||
import org.opensearch.action.get.GetResponse;
|
||||
import org.opensearch.action.index.IndexRequest;
|
||||
import org.opensearch.action.index.IndexResponse;
|
||||
import org.opensearch.action.support.master.AcknowledgedResponse;
|
||||
import org.opensearch.client.RequestOptions;
|
||||
import org.opensearch.client.RestClient;
|
||||
import org.opensearch.client.RestClientBuilder;
|
||||
import org.opensearch.client.RestHighLevelClient;
|
||||
import org.opensearch.client.indices.CreateIndexRequest;
|
||||
import org.opensearch.client.indices.CreateIndexResponse;
|
||||
import org.opensearch.common.settings.Settings;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
|
|
@ -59,10 +72,10 @@ public class RESTClientSample {
|
|||
|
||||
//Point to keystore with appropriate certificates for security.
|
||||
System.setProperty("javax.net.ssl.trustStore", "/full/path/to/keystore");
|
||||
System.setProperty("javax.net.ssl.trustStorePassword", password-to-keystore);
|
||||
System.setProperty("javax.net.ssl.trustStorePassword", "password-to-keystore");
|
||||
|
||||
//Establish credentials to use basic authentication.
|
||||
//Only for demo purposes. Do not specify your credentials in code.
|
||||
//Only for demo purposes. Don't specify your credentials in code.
|
||||
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
|
||||
credentialsProvider.setCredentials(AuthScope.ANY,
|
||||
|
|
@ -93,7 +106,7 @@ public class RESTClientSample {
|
|||
HashMap<String, Object> mapping = new HashMap<String, Object>();
|
||||
mapping.put("properties", ageMapping);
|
||||
createIndexRequest.mapping(mapping);
|
||||
CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest, RequestOptions.DEFAULT
|
||||
CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
|
||||
|
||||
//Adding data to the index.
|
||||
IndexRequest request = new IndexRequest("custom-index"); //Add a document to the custom-index we created.
|
||||
|
|
@ -122,3 +135,13 @@ public class RESTClientSample {
|
|||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Elasticsearch OSS Java high-level REST client
|
||||
|
||||
We recommend using the OpenSearch client to connect to OpenSearch clusters, but if you must use the Elasticsearch OSS Java high-level REST client, version 7.10.2 of the Elasticsearch OSS client also works with the 1.x versions of OpenSearch.
|
||||
|
||||
### Migrating to the OpenSearch Java high-level REST client
|
||||
|
||||
Migrating from the Elasticsearch OSS client to the OpenSearch high-level REST client is as simple as changing your Maven dependency to one that references [OpenSearch's dependency](#setup).
|
||||
|
||||
Afterward, change all references of `org.elasticsearch` to `org.opensearch`, and you're ready to start submitting requests to your OpenSearch cluster.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,302 @@
|
|||
---
|
||||
layout: default
|
||||
title: Java client
|
||||
nav_order: 65
|
||||
---
|
||||
|
||||
# Java client
|
||||
|
||||
The OpenSearch Java client is currently in its beta phase, so we recommend that you use the [OpenSearch Java high-level REST client]({{site.url}}{{site.baseurl}}/clients/java-rest-high-level).
|
||||
{: .note}
|
||||
|
||||
The OpenSearch Java client allows you to interact with your OpenSearch clusters through Java methods and data structures rather than HTTP methods and raw JSON.
|
||||
|
||||
For example, you can submit requests to your cluster using objects to create indices, add data to documents, or complete some other operation using the client's built-in methods.
|
||||
|
||||
## Install the client
|
||||
|
||||
To start using the OpenSearch Java client, ensure that you have the following dependencies in your project's `pom.xml` file:
|
||||
|
||||
```
|
||||
<dependency>
|
||||
<groupId>org.opensearch.client</groupId>
|
||||
<artifactId>opensearch-rest-client</artifactId>
|
||||
<version>{{site.opensearch_version}}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.opensearch.client</groupId>
|
||||
<artifactId>opensearch-java</artifactId>
|
||||
<version>0.1.0</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
If you're using Gradle, add the following dependencies to your project.
|
||||
|
||||
```
|
||||
dependencies {
|
||||
implementation 'org.opensearch.client:opensearch-rest-client: {{site.opensearch_version}}'
|
||||
implementation 'org.opensearch.client:opensearch-java:0.1.0'
|
||||
}
|
||||
```
|
||||
|
||||
You can now start your OpenSearch cluster.
|
||||
|
||||
## Security
|
||||
|
||||
Before using the REST client in your Java application, you must configure the application's truststore to connect to the security plugin. If you are using self-signed certificates or demo configurations, you can use the following command to create a custom truststore and add in root authority certificates.
|
||||
|
||||
If you're using certificates from a trusted Certificate Authority (CA), you don't need to configure the truststore.
|
||||
|
||||
```bash
|
||||
keytool -import <path-to-cert> -alias <alias-to-call-cert> -keystore <truststore-name>
|
||||
```
|
||||
|
||||
You can now point your Java client to the truststore and set basic authentication credentials that can access a secure cluster (refer to the sample code below on how to do so).
|
||||
|
||||
If you run into issues when configuring security, see [common issues]({{site.url}}{{site.baseurl}}/troubleshoot/index) and [troubleshoot TLS]({{site.url}}{{site.baseurl}}/troubleshoot/tls).
|
||||
|
||||
## Sample data
|
||||
|
||||
This section uses a class called `IndexData`, which is a simple Java class that stores basic data and methods. For your own OpenSearch cluster, you might find that you need a more robust class to store your data.
|
||||
|
||||
### IndexData class
|
||||
|
||||
```java
|
||||
static class IndexData {
|
||||
private String firstName;
|
||||
private String lastName;
|
||||
|
||||
public IndexData(String firstName, String lastName) {
|
||||
this.firstName = firstName;
|
||||
this.lastName = lastName;
|
||||
}
|
||||
|
||||
public String getFirstName() {
|
||||
return firstName;
|
||||
}
|
||||
|
||||
public void setFirstName(String firstName) {
|
||||
this.firstName = firstName;
|
||||
}
|
||||
|
||||
public String getLastName() {
|
||||
return lastName;
|
||||
}
|
||||
|
||||
public void setLastName(String lastName) {
|
||||
this.lastName = lastName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return String.format("IndexData{first name='%s', last name='%s'}", firstName, lastName);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Initialize the client with SSL and TLS enabled
|
||||
|
||||
This code example uses basic credentials that come with the default OpenSearch configuration. If you’re using the Java client with your own OpenSearch cluster, be sure to change the code to use your own credentials.
|
||||
|
||||
The following sample code initializes a client with SSL and TLS enabled:
|
||||
|
||||
```java
|
||||
import org.apache.http.HttpHost;
|
||||
import org.apache.http.auth.AuthScope;
|
||||
import org.apache.http.auth.UsernamePasswordCredentials;
|
||||
import org.apache.http.client.CredentialsProvider;
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider;
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
|
||||
import org.opensearch.client.RestClient;
|
||||
import org.opensearch.client.RestClientBuilder;
|
||||
import org.opensearch.clients.base.RestClientTransport;
|
||||
import org.opensearch.clients.base.Transport;
|
||||
import org.opensearch.clients.json.jackson.JacksonJsonpMapper;
|
||||
import org.opensearch.clients.opensearch.OpenSearchClient;
|
||||
import org.opensearch.clients.opensearch._global.IndexRequest;
|
||||
import org.opensearch.clients.opensearch._global.IndexResponse;
|
||||
import org.opensearch.clients.opensearch._global.SearchResponse;
|
||||
import org.opensearch.clients.opensearch.indices.*;
|
||||
import org.opensearch.clients.opensearch.indices.put_settings.IndexSettingsBody;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class OpenSearchClientExample {
|
||||
public static void main(String[] args) {
|
||||
try{
|
||||
System.setProperty("javax.net.ssl.trustStore", "/full/path/to/keystore");
|
||||
System.setProperty("javax.net.ssl.trustStorePassword", "password-to-keystore");
|
||||
|
||||
//Only for demo purposes. Don't specify your credentials in code.
|
||||
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY,
|
||||
new UsernamePasswordCredentials("admin", "admin"));
|
||||
|
||||
//Initialize the client with SSL and TLS enabled
|
||||
RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200, "https")).
|
||||
setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
|
||||
return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
}
|
||||
}).build();
|
||||
Transport transport = new RestClientTransport(restClient, new JacksonJsonpMapper());
|
||||
OpenSearchClient client = new OpenSearchClient(transport);
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## OpenSearch client examples
|
||||
|
||||
This section has sample code that shows you how to create an index with non-default settings, add a document to the index, search for the document, delete the document, and finally delete the index.
|
||||
|
||||
### Create an index with non-default settings
|
||||
|
||||
```java
|
||||
String index = "sample-index";
|
||||
CreateRequest createIndexRequest = new CreateRequest.Builder().index(index).build();
|
||||
client.indices().create(createIndexRequest);
|
||||
|
||||
IndexSettings indexSettings = new IndexSettings.Builder().autoExpandReplicas("0-all").build();
|
||||
IndexSettingsBody settingsBody = new IndexSettingsBody.Builder().settings(indexSettings).build();
|
||||
PutSettingsRequest putSettingsRequest = new PutSettingsRequest.Builder().index(index).value(settingsBody).build();
|
||||
client.indices().putSettings(putSettingsRequest);
|
||||
```
|
||||
|
||||
### Index data
|
||||
|
||||
```java
|
||||
IndexData indexData = new IndexData("first_name", "Bruce");
|
||||
IndexRequest<IndexData> indexRequest = new IndexRequest.Builder<IndexData>().index(index).id("1").value(indexData).build();
|
||||
client.index(indexRequest);
|
||||
```
|
||||
|
||||
### Search for the document
|
||||
|
||||
```java
|
||||
SearchResponse<IndexData> searchResponse = client.search(s -> s.index(index), IndexData.class);
|
||||
for (int i = 0; i< searchResponse.hits().hits().size(); i++) {
|
||||
System.out.println(searchResponse.hits().hits().get(i).source());
|
||||
}
|
||||
```
|
||||
|
||||
### Delete the document
|
||||
|
||||
The following sample code deletes a document whose ID is 1.
|
||||
|
||||
```java
|
||||
client.delete(b -> b.index(index).id("1"));
|
||||
```
|
||||
|
||||
### Delete the index
|
||||
|
||||
```java
|
||||
DeleteRequest deleteRequest = new DeleteRequest.Builder().index(index).build();
|
||||
DeleteResponse deleteResponse = client.indices().delete(deleteRequest);
|
||||
|
||||
restClient.close();
|
||||
} catch (IOException e){
|
||||
System.out.println(e.toString());
|
||||
} finally {
|
||||
try {
|
||||
if (client != null) {
|
||||
client.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
System.out.println(e.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Complete code sample
|
||||
|
||||
```java
|
||||
import org.apache.http.HttpHost;
|
||||
import org.apache.http.auth.AuthScope;
|
||||
import org.apache.http.auth.UsernamePasswordCredentials;
|
||||
import org.apache.http.client.CredentialsProvider;
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider;
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
|
||||
import org.opensearch.client.RestClient;
|
||||
import org.opensearch.client.RestClientBuilder;
|
||||
import org.opensearch.clients.base.RestClientTransport;
|
||||
import org.opensearch.clients.base.Transport;
|
||||
import org.opensearch.clients.json.jackson.JacksonJsonpMapper;
|
||||
import org.opensearch.clients.opensearch.OpenSearchClient;
|
||||
import org.opensearch.clients.opensearch._global.IndexRequest;
|
||||
import org.opensearch.clients.opensearch._global.IndexResponse;
|
||||
import org.opensearch.clients.opensearch._global.SearchResponse;
|
||||
import org.opensearch.clients.opensearch.indices.*;
|
||||
import org.opensearch.clients.opensearch.indices.put_settings.IndexSettingsBody;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class OpenSearchClientExample {
|
||||
public static void main(String[] args) {
|
||||
try{
|
||||
System.setProperty("javax.net.ssl.trustStore", "/full/path/to/keystore");
|
||||
System.setProperty("javax.net.ssl.trustStorePassword", "password-to-keystore");
|
||||
|
||||
//Only for demo purposes. Don't specify your credentials in code.
|
||||
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY,
|
||||
new UsernamePasswordCredentials("admin", "admin"));
|
||||
|
||||
//Initialize the client with SSL and TLS enabled
|
||||
RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200, "https")).
|
||||
setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
|
||||
return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
}
|
||||
}).build();
|
||||
Transport transport = new RestClientTransport(restClient, new JacksonJsonpMapper());
|
||||
OpenSearchClient client = new OpenSearchClient(transport);
|
||||
|
||||
//Create the index
|
||||
String index = "sample-index";
|
||||
CreateRequest createIndexRequest = new CreateRequest.Builder().index(index).build();
|
||||
client.indices().create(createIndexRequest);
|
||||
|
||||
//Add some settings to the index
|
||||
IndexSettings indexSettings = new IndexSettings.Builder().autoExpandReplicas("0-all").build();
|
||||
IndexSettingsBody settingsBody = new IndexSettingsBody.Builder().settings(indexSettings).build();
|
||||
PutSettingsRequest putSettingsRequest = new PutSettingsRequest.Builder().index(index).value(settingsBody).build();
|
||||
client.indices().putSettings(putSettingsRequest);
|
||||
|
||||
//Index some data
|
||||
IndexData indexData = new IndexData("first_name", "Bruce");
|
||||
IndexRequest<IndexData> indexRequest = new IndexRequest.Builder<IndexData>().index(index).id("1").value(indexData).build();
|
||||
client.index(indexRequest);
|
||||
|
||||
//Search for the document
|
||||
SearchResponse<IndexData> searchResponse = client.search(s -> s.index(index), IndexData.class);
|
||||
for (int i = 0; i< searchResponse.hits().hits().size(); i++) {
|
||||
System.out.println(searchResponse.hits().hits().get(i).source());
|
||||
}
|
||||
|
||||
//Delete the document
|
||||
client.delete(b -> b.index(index).id("1"));
|
||||
|
||||
// Delete the index
|
||||
DeleteRequest deleteRequest = new DeleteRequest.Builder().index(index).build();
|
||||
DeleteResponse deleteResponse = client.indices().delete(deleteRequest);
|
||||
|
||||
restClient.close();
|
||||
} catch (IOException e){
|
||||
System.out.println(e.toString());
|
||||
} finally {
|
||||
try {
|
||||
if (client != null) {
|
||||
client.close();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
System.out.println(e.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
|
@ -0,0 +1,141 @@
|
|||
---
|
||||
layout: default
|
||||
title: JavaScript client
|
||||
nav_order: 90
|
||||
---
|
||||
|
||||
# JavaScript client
|
||||
|
||||
The OpenSearch JavaScript client provides a safer and easier way to interact with your OpenSearch cluster. Rather than using OpenSearch from the browser and potentially exposing your data to the public, you can build an OpenSearch client that takes care of sending requests to your cluster.
|
||||
|
||||
The client contains a library of APIs that let you perform different operations on your cluster and return a standard response body. The example here demonstrates some basic operations like creating an index, adding documents, and searching your data.
|
||||
|
||||
## Setup
|
||||
|
||||
To add the client to your project, install it from [npm](https://www.npmjs.com):
|
||||
|
||||
```bash
|
||||
npm install @opensearch-project/opensearch
|
||||
```
|
||||
|
||||
To install a specific major version of the client, run the following command:
|
||||
|
||||
```bash
|
||||
npm install @opensearch-project/opensearch@<version>
|
||||
```
|
||||
|
||||
If you prefer to add the client manually or just want to examine the source code, see [opensearch-js](https://github.com/opensearch-project/opensearch-js) on GitHub.
|
||||
|
||||
Then require the client:
|
||||
|
||||
```javascript
|
||||
const { Client } = require("@opensearch-project/opensearch");
|
||||
```
|
||||
|
||||
## Sample code
|
||||
|
||||
```javascript
|
||||
"use strict";
|
||||
|
||||
var host = "localhost";
|
||||
var protocol = "https";
|
||||
var port = 9200;
|
||||
var auth = "admin:admin"; // For testing only. Don't store credentials in code.
|
||||
var ca_certs_path = "/full/path/to/root-ca.pem";
|
||||
|
||||
// Optional client certificates if you don't want to use HTTP basic authentication.
|
||||
// var client_cert_path = '/full/path/to/client.pem'
|
||||
// var client_key_path = '/full/path/to/client-key.pem'
|
||||
|
||||
// Create a client with SSL/TLS enabled.
|
||||
var { Client } = require("@opensearch-project/opensearch");
|
||||
var fs = require("fs");
|
||||
var client = new Client({
|
||||
node: protocol + "://" + auth + "@" + host + ":" + port,
|
||||
ssl: {
|
||||
ca: fs.readFileSync(ca_certs_path),
|
||||
// You can turn off certificate verification (rejectUnauthorized: false) if you're using self-signed certificates with a hostname mismatch.
|
||||
// cert: fs.readFileSync(client_cert_path),
|
||||
// key: fs.readFileSync(client_key_path)
|
||||
},
|
||||
});
|
||||
|
||||
async function search() {
|
||||
// Create an index with non-default settings.
|
||||
var index_name = "books";
|
||||
var settings = {
|
||||
settings: {
|
||||
index: {
|
||||
number_of_shards: 4,
|
||||
number_of_replicas: 3,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
var response = await client.indices.create({
|
||||
index: index_name,
|
||||
body: settings,
|
||||
});
|
||||
|
||||
console.log("Creating index:");
|
||||
console.log(response.body);
|
||||
|
||||
// Add a document to the index.
|
||||
var document = {
|
||||
title: "The Outsider",
|
||||
author: "Stephen King",
|
||||
year: "2018",
|
||||
genre: "Crime fiction",
|
||||
};
|
||||
|
||||
var id = "1";
|
||||
|
||||
var response = await client.index({
|
||||
id: id,
|
||||
index: index_name,
|
||||
body: document,
|
||||
refresh: true,
|
||||
});
|
||||
|
||||
console.log("Adding document:");
|
||||
console.log(response.body);
|
||||
|
||||
// Search for the document.
|
||||
var query = {
|
||||
query: {
|
||||
match: {
|
||||
title: {
|
||||
query: "The Outsider",
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
var response = await client.search({
|
||||
index: index_name,
|
||||
body: query,
|
||||
});
|
||||
|
||||
console.log("Search results:");
|
||||
console.log(response.body.hits);
|
||||
|
||||
// Delete the document.
|
||||
var response = await client.delete({
|
||||
index: index_name,
|
||||
id: id,
|
||||
});
|
||||
|
||||
console.log("Deleting document:");
|
||||
console.log(response.body);
|
||||
|
||||
// Delete the index.
|
||||
var response = await client.indices.delete({
|
||||
index: index_name,
|
||||
});
|
||||
|
||||
console.log("Deleting index:");
|
||||
console.log(response.body);
|
||||
}
|
||||
|
||||
search().catch(console.log);
|
||||
```
|
||||
|
|
@ -57,6 +57,9 @@ The OpenSearch Logstash plugin has two installation options at this time: Linux
|
|||
|
||||
Make sure you have [Java Development Kit (JDK)](https://www.oracle.com/java/technologies/javase-downloads.html) version 8 or 11 installed.
|
||||
|
||||
If you're migrating from an existing Logstash installation, you can install the [OpenSearch output plugin](https://rubygems.org/gems/logstash-output-opensearch/) manually and [update pipeline.conf](https://opensearch.org/docs/latest/clients/logstash/ship-to-opensearch/). We include this plugin by default in our tarball and Docker downloads.
|
||||
{: .note }
|
||||
|
||||
### Tarball
|
||||
|
||||
1. Download the Logstash tarball from [OpenSearch downloads](https://opensearch.org/downloads.html).
|
||||
|
|
@ -64,10 +67,10 @@ Make sure you have [Java Development Kit (JDK)](https://www.oracle.com/java/tech
|
|||
2. Navigate to the downloaded folder in the terminal and extract the files:
|
||||
|
||||
```bash
|
||||
tar -zxvf logstash-oss-with-opensearch-output-plugin-7.13.2-linux-x64.tar.gz
|
||||
tar -zxvf logstash-oss-with-opensearch-output-plugin-7.16.2-linux-x64.tar.gz
|
||||
```
|
||||
|
||||
3. Navigate to the `logstash-7.13.2` directory.
|
||||
3. Navigate to the `logstash-7.16.2` directory.
|
||||
- You can add your pipeline configurations to the `config` directory. Logstash saves any data from the plugins in the `data` directory. The `bin` directory contains the binaries for starting Logstash and managing plugins.
|
||||
|
||||
### Docker
|
||||
|
|
@ -75,7 +78,7 @@ Make sure you have [Java Development Kit (JDK)](https://www.oracle.com/java/tech
|
|||
1. Pull the Logstash oss package with the OpenSearch output plugin image:
|
||||
|
||||
```
|
||||
docker pull opensearchproject/logstash-oss-with-opensearch-output-plugin:7.13.2
|
||||
docker pull opensearchproject/logstash-oss-with-opensearch-output-plugin:7.16.2
|
||||
```
|
||||
|
||||
1. Create a Docker network:
|
||||
|
|
@ -87,13 +90,13 @@ Make sure you have [Java Development Kit (JDK)](https://www.oracle.com/java/tech
|
|||
1. Start OpenSearch with this network:
|
||||
|
||||
```
|
||||
docker run -p 9200:9200 -p 9600:9600 --name opensearch --net test -e "discovery.type=single-node" opensearchproject/opensearch:1.0.0
|
||||
docker run -p 9200:9200 -p 9600:9600 --name opensearch --net test -e "discovery.type=single-node" opensearchproject/opensearch:1.2.0
|
||||
```
|
||||
|
||||
1. Start Logstash:
|
||||
|
||||
```
|
||||
docker run -it --rm --name logstash --net test opensearchproject/logstash-oss-with-opensearch-output-plugin:7.13.2 -e 'input { stdin { } } output {
|
||||
docker run -it --rm --name logstash --net test opensearchproject/logstash-oss-with-opensearch-output-plugin:7.13.4 -e 'input { stdin { } } output {
|
||||
opensearch {
|
||||
hosts => ["https://opensearch:9200"]
|
||||
index => "opensearch-logstash-docker-%{+YYYY.MM.dd}"
|
||||
|
|
|
|||
|
|
@ -75,3 +75,72 @@ output {
|
|||
health | status | index | uuid | pri | rep | docs.count | docs.deleted | store.size | pri.store.size
|
||||
green | open | logstash-logs-2021.07.01 | iuh648LYSnmQrkGf70pplA | 1 | 1 | 1 | 0 | 10.3kb | 5.1kb
|
||||
```
|
||||
|
||||
## Adding different Authentication mechanisms in the Output plugin
|
||||
|
||||
## auth_type to support different authentication mechanisms
|
||||
|
||||
In addition to the existing authentication mechanisms, if we want to add new authentication then we will be adding them in the configuration by using auth_type
|
||||
|
||||
Example Configuration for basic authentication:
|
||||
|
||||
```yml
|
||||
output {
|
||||
opensearch {
|
||||
hosts => ["https://hostname:port"]
|
||||
auth_type => {
|
||||
type => 'basic'
|
||||
user => 'admin'
|
||||
password => 'admin'
|
||||
}
|
||||
index => "logstash-logs-%{+YYYY.MM.dd}"
|
||||
}
|
||||
}
|
||||
```
|
||||
### Parameters inside auth_type
|
||||
|
||||
- type (string) - We should specify the type of authentication
|
||||
- We should add credentials required for that authentication like 'user' and 'password' for 'basic' authentication
|
||||
- We should also add other parameters required for that authentication mechanism like we added 'region' for 'aws_iam' authentication
|
||||
|
||||
## Configuration for AWS IAM Authentication
|
||||
|
||||
To run the Logstash Output Opensearch plugin using aws_iam authentication, simply add a configuration following the below documentation.
|
||||
|
||||
Example Configuration:
|
||||
|
||||
```yml
|
||||
output {
|
||||
opensearch {
|
||||
hosts => ["https://hostname:port"]
|
||||
auth_type => {
|
||||
type => 'aws_iam'
|
||||
aws_access_key_id => 'ACCESS_KEY'
|
||||
aws_secret_access_key => 'SECRET_KEY'
|
||||
region => 'us-west-2'
|
||||
}
|
||||
index => "logstash-logs-%{+YYYY.MM.dd}"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Required Parameters
|
||||
|
||||
- hosts (array of string) - AmazonOpensearchService domain endpoint : port number
|
||||
- auth_type (Json object) - Which holds other parameters required for authentication
|
||||
- type (string) - "aws_iam"
|
||||
- aws_access_key_id (string) - AWS access key
|
||||
- aws_secret_access_key (string) - AWS secret access key
|
||||
- region (string, :default => "us-east-1") - region in which the domain is located
|
||||
- if we want to pass other optional parameters like profile, session_token,etc. They needs to be added in auth_type
|
||||
- port (string) - AmazonOpensearchService listens on port 443 for HTTPS
|
||||
- protocol (string) - The protocol used to connect to AmazonOpensearchService is 'https'
|
||||
|
||||
### Optional Parameters
|
||||
- The credential resolution logic can be described as follows:
|
||||
- User passed aws_access_key_id and aws_secret_access_key in configuration
|
||||
- Environment variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY (RECOMMENDED since they are recognized by all the AWS SDKs and CLI except for .NET), or AWS_ACCESS_KEY and AWS_SECRET_KEY (only recognized by Java SDK)
|
||||
- Credential profiles file at the default location (~/.aws/credentials) shared by all AWS SDKs and the AWS CLI
|
||||
- Instance profile credentials delivered through the Amazon EC2 metadata service
|
||||
- template (path) - You can set the path to your own template here, if you so desire. If not set, the included template will be used.
|
||||
- template_name (string, default => "logstash") - defines how the template is named inside Opensearch
|
||||
|
|
|
|||
|
|
@ -0,0 +1,100 @@
|
|||
---
|
||||
layout: default
|
||||
title: PHP client
|
||||
nav_order: 90
|
||||
---
|
||||
|
||||
# PHP client
|
||||
|
||||
The OpenSearch PHP client provides a safer and easier way to interact with your OpenSearch cluster. Rather than using OpenSearch from the browser and potentially exposing your data to the public, you can build an OpenSearch client that takes care of sending requests to your cluster.
|
||||
|
||||
The client contains a library of APIs that let you perform different operations on your cluster and return a standard response body. The example here demonstrates some basic operations like creating an index, adding documents, and searching your data.
|
||||
|
||||
## Setup
|
||||
|
||||
To add the client to your project, install it using [composer](https://getcomposer.org/):
|
||||
|
||||
```bash
|
||||
composer require opensearch-project/opensearch-php
|
||||
```
|
||||
|
||||
To install a specific major version of the client, run the following command:
|
||||
|
||||
```bash
|
||||
composer require opensearch-project/opensearch-php:<version>
|
||||
```
|
||||
|
||||
Then require the autload file from composer in your code:
|
||||
|
||||
```php
|
||||
require __DIR__ . '/vendor/autoload.php';
|
||||
```
|
||||
|
||||
## Sample code
|
||||
|
||||
```php
|
||||
<?php
|
||||
|
||||
require __DIR__ . '/vendor/autoload.php';
|
||||
|
||||
$client = (new \OpenSearch\ClientBuilder())
|
||||
->setHosts(['https://localhost:9200'])
|
||||
->setBasicAuthentication('admin', 'admin') // For testing only. Don't store credentials in code.
|
||||
->setSSLVerification(false) // For testing only. Use certificate for validation
|
||||
->build();
|
||||
|
||||
$indexName = 'test-index-name';
|
||||
|
||||
// Print OpenSearch version information on console.
|
||||
var_dump($client->info());
|
||||
|
||||
// Create an index with non-default settings.
|
||||
$client->indices()->create([
|
||||
'index' => $indexName,
|
||||
'body' => [
|
||||
'settings' => [
|
||||
'index' => [
|
||||
'number_of_shards' => 4
|
||||
]
|
||||
]
|
||||
]
|
||||
]);
|
||||
|
||||
$client->create([
|
||||
'index' => $indexName,
|
||||
'id' => 1,
|
||||
'body' => [
|
||||
'title' => 'Moneyball',
|
||||
'director' => 'Bennett Miller',
|
||||
'year' => 2011
|
||||
]
|
||||
]);
|
||||
|
||||
// Search for it
|
||||
var_dump(
|
||||
$client->search([
|
||||
'index' => $indexName,
|
||||
'body' => [
|
||||
'size' => 5,
|
||||
'query' => [
|
||||
'multi_match' => [
|
||||
'query' => 'miller',
|
||||
'fields' => ['title^2', 'director']
|
||||
]
|
||||
]
|
||||
]
|
||||
])
|
||||
);
|
||||
|
||||
// Delete a single document
|
||||
$client->delete([
|
||||
'index' => $indexName,
|
||||
'id' => 1,
|
||||
]);
|
||||
|
||||
|
||||
// Delete index
|
||||
$client->indices()->delete([
|
||||
'index' => $indexName
|
||||
]);
|
||||
```
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
---
|
||||
layout: default
|
||||
title: Python client
|
||||
nav_order: 70
|
||||
---
|
||||
|
||||
# Python client
|
||||
|
||||
The OpenSearch Python client provides a more natural syntax for interacting with your cluster. Rather than sending HTTP requests to a given URL, you can create an OpenSearch client for your cluster and call the client's built-in functions.
|
||||
|
||||
{% comment %}
|
||||
`opensearch-py` is the lower-level of the two Python clients. If you want a general client for assorted operations, it's a great choice. If you want a higher-level client strictly for indexing and search operations, consider [opensearch-dsl-py]({{site.url}}{{site.baseurl}}/clients/python-dsl/).
|
||||
{% endcomment %}
|
||||
|
||||
|
||||
## Setup
|
||||
|
||||
To add the client to your project, install it using [pip](https://pip.pypa.io/):
|
||||
|
||||
```bash
|
||||
pip install opensearch-py
|
||||
```
|
||||
|
||||
Then import it like any other module:
|
||||
|
||||
```python
|
||||
from opensearchpy import OpenSearch
|
||||
```
|
||||
|
||||
If you prefer to add the client manually or just want to examine the source code, see [opensearch-py on GitHub](https://github.com/opensearch-project/opensearch-py).
|
||||
|
||||
|
||||
## Sample code
|
||||
|
||||
```python
|
||||
from opensearchpy import OpenSearch
|
||||
|
||||
host = 'localhost'
|
||||
port = 9200
|
||||
auth = ('admin', 'admin') # For testing only. Don't store credentials in code.
|
||||
ca_certs_path = '/full/path/to/root-ca.pem' # Provide a CA bundle if you use intermediate CAs with your root CA.
|
||||
|
||||
# Optional client certificates if you don't want to use HTTP basic authentication.
|
||||
# client_cert_path = '/full/path/to/client.pem'
|
||||
# client_key_path = '/full/path/to/client-key.pem'
|
||||
|
||||
# Create the client with SSL/TLS enabled, but hostname verification disabled.
|
||||
client = OpenSearch(
|
||||
hosts = [{'host': host, 'port': port}],
|
||||
http_compress = True, # enables gzip compression for request bodies
|
||||
http_auth = auth,
|
||||
# client_cert = client_cert_path,
|
||||
# client_key = client_key_path,
|
||||
use_ssl = True,
|
||||
verify_certs = True,
|
||||
ssl_assert_hostname = False,
|
||||
ssl_show_warn = False,
|
||||
ca_certs = ca_certs_path
|
||||
)
|
||||
|
||||
# Create an index with non-default settings.
|
||||
index_name = 'python-test-index'
|
||||
index_body = {
|
||||
'settings': {
|
||||
'index': {
|
||||
'number_of_shards': 4
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
response = client.indices.create(index_name, body=index_body)
|
||||
print('\nCreating index:')
|
||||
print(response)
|
||||
|
||||
# Add a document to the index.
|
||||
document = {
|
||||
'title': 'Moneyball',
|
||||
'director': 'Bennett Miller',
|
||||
'year': '2011'
|
||||
}
|
||||
id = '1'
|
||||
|
||||
response = client.index(
|
||||
index = index_name,
|
||||
body = document,
|
||||
id = id,
|
||||
refresh = True
|
||||
)
|
||||
|
||||
print('\nAdding document:')
|
||||
print(response)
|
||||
|
||||
# Search for the document.
|
||||
q = 'miller'
|
||||
query = {
|
||||
'size': 5,
|
||||
'query': {
|
||||
'multi_match': {
|
||||
'query': q,
|
||||
'fields': ['title^2', 'director']
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
response = client.search(
|
||||
body = query,
|
||||
index = index_name
|
||||
)
|
||||
print('\nSearch results:')
|
||||
print(response)
|
||||
|
||||
# Delete the document.
|
||||
response = client.delete(
|
||||
index = index_name,
|
||||
id = id
|
||||
)
|
||||
|
||||
print('\nDeleting document:')
|
||||
print(response)
|
||||
|
||||
# Delete the index.
|
||||
response = client.indices.delete(
|
||||
index = index_name
|
||||
)
|
||||
|
||||
print('\nDeleting index:')
|
||||
print(response)
|
||||
```
|
||||
32
_config.yml
32
_config.yml
|
|
@ -1,13 +1,14 @@
|
|||
title: OpenSearch documentation
|
||||
description: >- # this means to ignore newlines until "baseurl:"
|
||||
Documentation for OpenSearch, the Apache 2.0 search, analytics, and visualization suite with advanced security, alerting, SQL support, automated index management, deep performance analysis, and more.
|
||||
baseurl: "/docs" # the subpath of your site, e.g. /blog
|
||||
baseurl: "/docs/latest" # the subpath of your site, e.g. /blog
|
||||
url: "https://opensearch.org" # the base hostname & protocol for your site, e.g. http://example.com
|
||||
permalink: /:path/
|
||||
|
||||
opensearch_version: 1.0.1
|
||||
opensearch_major_minor_version: 1.0
|
||||
lucene_version: 8_8_2
|
||||
opensearch_version: 1.2.3
|
||||
opensearch_dashboards_version: 1.2.0
|
||||
opensearch_major_minor_version: 1.2
|
||||
lucene_version: 8_9_0
|
||||
|
||||
# Build settings
|
||||
markdown: kramdown
|
||||
|
|
@ -45,6 +46,12 @@ collections:
|
|||
im-plugin:
|
||||
permalink: /:collection/:path/
|
||||
output: true
|
||||
replication-plugin:
|
||||
permalink: /:collection/:path/
|
||||
output: true
|
||||
observability:
|
||||
permalink: /:collection/:path/
|
||||
output: true
|
||||
monitoring-plugins:
|
||||
permalink: /:collection/:path/
|
||||
output: true
|
||||
|
|
@ -81,6 +88,12 @@ just_the_docs:
|
|||
im-plugin:
|
||||
name: Index management plugin
|
||||
nav_fold: true
|
||||
replication-plugin:
|
||||
name: Replication plugin
|
||||
nav_fold: true
|
||||
observability:
|
||||
name: Observability
|
||||
nav_fold: true
|
||||
monitoring-plugins:
|
||||
name: Monitoring plugins
|
||||
nav_fold: true
|
||||
|
|
@ -95,8 +108,15 @@ just_the_docs:
|
|||
|
||||
|
||||
# Enable or disable the site search
|
||||
# Supports true (default) or false
|
||||
search_enabled: true
|
||||
# By default, just-the-docs enables its JSON file-based search. We also have an OpenSearch-driven search functionality.
|
||||
# To disable any search from appearing, both `search_enabled` and `use_custom_search` need to be false.
|
||||
# To use the OpenSearch-driven search, `search_enabled` has to be false and `use_custom_search` needs to be true.
|
||||
# If `search_enabled` is true, irrespective of the value of `use_custom_search`, the JSON file-based search appears.
|
||||
#
|
||||
# `search_enabled` defaults to true
|
||||
# `use_custom_search` defaults to false
|
||||
search_enabled: false
|
||||
use_custom_search: true
|
||||
|
||||
search:
|
||||
# Split pages into sections that can be searched individually
|
||||
|
|
|
|||
|
|
@ -0,0 +1,91 @@
|
|||
---
|
||||
layout: default
|
||||
title: Custom branding
|
||||
nav_order: 200
|
||||
---
|
||||
|
||||
# OpenSearch Dashboards custom branding
|
||||
Introduced 1.2
|
||||
{: .label .label-purple }
|
||||
|
||||
By default, OpenSearch Dashboards uses the OpenSearch logo, but if you want to use custom branding elements such as the favicon or main Dashboards logo, you can do so by editing `opensearch_dashboards.yml` or by including a custom `opensearch_dashboards.yml` file when you start your OpenSearch cluster.
|
||||
|
||||
For example, if you're using Docker to start your OpenSearch cluster, include the following lines in the `opensearch-dashboards` section of your `docker-compose.yml` file:
|
||||
|
||||
```
|
||||
volumes:
|
||||
- ./opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml
|
||||
```
|
||||
|
||||
Doing so replaces the Docker image's default `opensearch_dashboards.yml` with your custom `opensearch_dashboards.yml` file, so be sure to include your desired settings as well. For example, if you want to configure TLS for OpenSearch Dashboards, see [Configure TLS for OpenSearch Dashboards]({{site.url}}{{site.baseurl}}/dashboards/install/tls).
|
||||
|
||||
Re-launch OpenSearch Dashboards, and OpenSearch Dashboards now uses your custom elements.
|
||||
|
||||
The following elements in OpenSearch Dashboards are customizable:
|
||||
|
||||
|
||||
|
||||
Setting | Corresponding branding element
|
||||
:--- | :---
|
||||
logo | Header logo. See #1 in the image.
|
||||
mark | OpenSearch Dashboards mark. See #2 in the image.
|
||||
loadingLogo | Loading logo used when OpenSearch Dashboards is starting. See #3 in the image.
|
||||
faviconUrl | Website icon. Loads next to the application title. See #4 in the image.
|
||||
applicationTitle | The application's title. See #5 in the image.
|
||||
|
||||
To start using your own branding elements in OpenSearch Dashboards, first uncomment this section of `opensearch_dashboards.yml`:
|
||||
|
||||
```yml
|
||||
# opensearchDashboards.branding:
|
||||
# logo:
|
||||
# defaultUrl: ""
|
||||
# darkModeUrl: ""
|
||||
# mark:
|
||||
# defaultUrl: ""
|
||||
# darkModeUrl: ""
|
||||
# loadingLogo:
|
||||
# defaultUrl: ""
|
||||
# darkModeUrl: ""
|
||||
# faviconUrl: ""
|
||||
# applicationTitle: ""
|
||||
```
|
||||
|
||||
Add the URLs you want to use as branding elements to the appropriate setting. Valid image types are `SVG`, `PNG`, and `GIF`.
|
||||
|
||||
Customization of dark mode Dashboards is also available, but you first must supply a valid link to `defaultUrl`, and then link to your preferred image with `darkModeUrl`. You are not required to customize all branding elements, so if you wanted to, it's perfectly valid to change just the logo. Leave unchanged elements as commented.
|
||||
|
||||
The following example demonstrates how to use `SVG` files as logos but leaves the other elements as defaults.
|
||||
|
||||
```yml
|
||||
logo:
|
||||
defaultUrl: "https://example.com/validUrl.svg"
|
||||
darkModeUrl: "https://example.com/validDarkModeUrl.svg"
|
||||
# mark:
|
||||
# defaultUrl: ""
|
||||
# darkModeUrl: ""
|
||||
# loadingLogo:
|
||||
# defaultUrl: ""
|
||||
# darkModeUrl: ""
|
||||
# faviconUrl: ""
|
||||
applicationTitle: "My custom application"
|
||||
```
|
||||
|
||||
We recommend linking to images that are hosted on a web server, but if you really want to use locally hosted images, save your images inside `src/core/server/core_app/assets`, and then configure `opensearch_dashboards.yml`. You can access locally stored images through the `ui` folder.
|
||||
|
||||
The following example assumes the default port of 5601 that Dashboards uses and demonstrates how to link to locally stored images.
|
||||
|
||||
```yml
|
||||
logo:
|
||||
defaultUrl: "https://localhost:5601/ui/my-own-image.svg"
|
||||
darkModeUrl: "https://localhost:5601/ui/my-own-image.svg"
|
||||
mark:
|
||||
defaultUrl: "https://localhost:5601/ui/my-own-image2.svg"
|
||||
darkModeUrl: "https://localhost:5601/ui/my-own-image2.svg"
|
||||
# loadingLogo:
|
||||
# defaultUrl: ""
|
||||
# darkModeUrl: ""
|
||||
# faviconUrl: ""
|
||||
applicationTitle: "My custom application"
|
||||
```
|
||||
|
||||
We don't recommend this workaround because new versions of Dashboards would revert all customized elements back to default branding elements, and you would have to re-upload your assets to access them again.
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
---
|
||||
layout: default
|
||||
title: Dashboards query language
|
||||
nav_order: 99
|
||||
---
|
||||
|
||||
# Dashboards Query Language
|
||||
|
||||
Similar to the [Query DSL]({{site.url}}{{site.baseurl}}/opensearch/query-dsl/index) that lets you use the HTTP request body to search for data, you can use the Dashbaords Query Language (DQL) in OpenSearch Dashboards to search for data and visualizations.
|
||||
|
||||
For example, if you want to see all visualizations of visits to a host based in the US, enter `geo.dest:US` into the search field, and Dashboards refreshes to display all related data.
|
||||
|
||||
Just like the query DSL, DQL has a handful of query types, so use whichever best fits your use case.
|
||||
|
||||
This section uses the OpenSearch Dashboards sample web log data. To add sample data in Dashboards, log in to OpenSearch Dashboards, choose **Home**, **Add sample data**, and then **Add data**.
|
||||
|
||||
---
|
||||
|
||||
#### Table of contents
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Terms query
|
||||
|
||||
The most basic query is to just specify the term you're searching for.
|
||||
|
||||
```
|
||||
host:www.example.com
|
||||
```
|
||||
|
||||
To access an object's nested field, list the complete path to the field separated by periods. For example, to retrieve the `lat` field in the `coordinates` object:
|
||||
|
||||
```
|
||||
coordinates.lat:43.7102
|
||||
```
|
||||
|
||||
DQL also supports leading and trailing wildcards, so you can search for any terms that match your pattern.
|
||||
|
||||
```
|
||||
host.keyword:*.example.com/*
|
||||
```
|
||||
|
||||
To check if a field exists or has any data, use a wildcard to see if Dashboards returns any results.
|
||||
|
||||
```
|
||||
host.keyword:*
|
||||
```
|
||||
|
||||
## Boolean query
|
||||
|
||||
To mix and match, or even combine, multiple queries for more refined results, you can use the boolean operators `and`, `or`, and `not`. DQL is not case sensitive, so `AND` and `and` are the same.
|
||||
|
||||
```
|
||||
host.keyword:www.example.com and response.keyword:200
|
||||
```
|
||||
|
||||
The following example demonstrates how to use multiple operators in one query.
|
||||
|
||||
```
|
||||
geo.dest:US or response.keyword:200 and host.keyword:www.example.com
|
||||
```
|
||||
|
||||
Remember that boolean operators follow the logical precedence order of `not`, `and`, and `or`, so if you have an expression like the previous example, `response.keyword:200 and host.keyword:www.example.com` gets evaluated first, and then Dashboards uses that result to compare with `geo.dest:US`.
|
||||
|
||||
To avoid confusion, we recommend using parentheses to dictate the order you want to evaluate in. If you want to evaluate `geo.dest:US or response.keyword:200` first, your expression becomes:
|
||||
|
||||
```
|
||||
(geo.dest:US or response.keyword:200) and host.keyword:www.example.com
|
||||
```
|
||||
|
||||
## Date and range queries
|
||||
|
||||
DQL also supports inequalities if you're using numeric inequalities.
|
||||
|
||||
```
|
||||
bytes >= 15 and memory < 15
|
||||
```
|
||||
|
||||
Similarly, you can use the same method to find a date before or after your query. `>` indicates a search for a date after your specified date, and `<` returns dates before.
|
||||
|
||||
```
|
||||
@timestamp > "2020-12-14T09:35:33"
|
||||
```
|
||||
|
||||
## Nested field query
|
||||
|
||||
If you have a document with nested fields, you have to specify which parts of the document you want to retrieve.
|
||||
|
||||
Suppose that you have the following document:
|
||||
|
||||
```json
|
||||
{
|
||||
"superheroes":[
|
||||
{
|
||||
"hero-name": "Superman",
|
||||
"real-identity": "Clark Kent",
|
||||
"age": 28
|
||||
},
|
||||
{
|
||||
"hero-name": "Batman",
|
||||
"real-identity": "Bruce Wayne",
|
||||
"age": 26
|
||||
},
|
||||
{
|
||||
"hero-name": "Flash",
|
||||
"real-identity": "Barry Allen",
|
||||
"age": 28
|
||||
},
|
||||
{
|
||||
"hero-name": "Robin",
|
||||
"real-identity": "Dick Grayson",
|
||||
"age": 15
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The following example demonstrates how to use DQL to retrieve a specific field.
|
||||
|
||||
```
|
||||
superheroes: {hero-name: Superman}
|
||||
```
|
||||
|
||||
If you want to retrieve multiple objects from your document, just specify all of the fields you want to retrieve.
|
||||
|
||||
```
|
||||
superheroes: {hero-name: Superman} and superheroes: {hero-name: Batman}
|
||||
```
|
||||
|
||||
The previous boolean and range queries still work, so you can submit a more refined query.
|
||||
|
||||
```
|
||||
superheroes: {hero-name: Superman and age < 50}
|
||||
```
|
||||
|
||||
If your document has an object nested within another object, you can still retrieve data by specifying all of the levels.
|
||||
|
||||
```
|
||||
justice-league.superheroes: {hero-name:Superman}
|
||||
```
|
||||
|
|
@ -9,7 +9,7 @@ nav_order: 1
|
|||
|
||||
You *can* start OpenSearch Dashboards using `docker run` after [creating a Docker network](https://docs.docker.com/engine/reference/commandline/network_create/) and starting OpenSearch, but the process of connecting OpenSearch Dashboards to OpenSearch is significantly easier with a Docker Compose file.
|
||||
|
||||
1. Run `docker pull opensearchproject/opensearch-dashboards:{{site.opensearch_version}}`.
|
||||
1. Run `docker pull opensearchproject/opensearch-dashboards:{{site.opensearch_dashboards_version}}`.
|
||||
|
||||
1. Create a [`docker-compose.yml`](https://docs.docker.com/compose/compose-file/) file appropriate for your environment. A sample file that includes OpenSearch Dashboards is available on the OpenSearch [Docker installation page]({{site.url}}{{site.baseurl}}/opensearch/install/docker#sample-docker-compose-file).
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ Resource | Description
|
|||
The specification in the default Helm chart supports many standard use cases and setups. You can modify the default chart to configure your desired specifications and set Transport Layer Security (TLS) and role-based access control (RBAC).
|
||||
|
||||
For information about the default configuration, steps to configure security, and configurable parameters, see the
|
||||
[README](https://github.com/opensearch-project/opensearch-devops/blob/main/Helm/README.md).
|
||||
[README](https://github.com/opensearch-project/helm-charts/tree/main/charts).
|
||||
|
||||
The instructions here assume you have a Kubernetes cluster with Helm preinstalled. See the [Kubernetes documentation](https://kubernetes.io/docs/setup/) for steps to configure a Kubernetes cluster and the [Helm documentation](https://helm.sh/docs/intro/install/) to install Helm.
|
||||
{: .note }
|
||||
|
|
|
|||
|
|
@ -28,6 +28,34 @@ If you don't want to use the all-in-one installation options, you can install th
|
|||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>1.2.0</td>
|
||||
<td>
|
||||
<pre>alertingDashboards 1.2.0.0
|
||||
anomalyDetectionDashboards 1.2.0.0
|
||||
ganttChartDashboards 1.2.0.0
|
||||
indexManagementDashboards 1.2.0.0
|
||||
queryWorkbenchDashboards 1.2.0.0
|
||||
reportsDashboards 1.2.0.0
|
||||
securityDashboards 1.2.0.0
|
||||
</pre>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>1.1.0</td>
|
||||
<td>
|
||||
<pre>alertingDashboards 1.1.0.0
|
||||
anomalyDetectionDashboards 1.1.0.0
|
||||
ganttChartDashboards 1.1.0.0
|
||||
indexManagementDashboards 1.1.0.0
|
||||
notebooksDashboards 1.1.0.0
|
||||
queryWorkbenchDashboards 1.1.0.0
|
||||
reportsDashboards 1.1.0.0
|
||||
securityDashboards 1.1.0.0
|
||||
traceAnalyticsDashboards 1.1.0.0
|
||||
</pre>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>1.0.1</td>
|
||||
<td>
|
||||
|
|
|
|||
|
|
@ -14,9 +14,10 @@ nav_order: 30
|
|||
```bash
|
||||
# x64
|
||||
tar -zxf opensearch-dashboards-{{site.opensearch_version}}-linux-x64.tar.gz
|
||||
cd opensearch-dashboards{% comment %}# ARM64
|
||||
cd opensearch-dashboards
|
||||
# ARM64
|
||||
tar -zxf opensearch-dashboards-{{site.opensearch_version}}-linux-arm64.tar.gz
|
||||
cd opensearch-dashboards{% endcomment %}
|
||||
cd opensearch-dashboards
|
||||
```
|
||||
|
||||
1. If desired, modify `config/opensearch_dashboards.yml`.
|
||||
|
|
@ -26,5 +27,3 @@ nav_order: 30
|
|||
```bash
|
||||
./bin/opensearch-dashboards
|
||||
```
|
||||
|
||||
1. See the [OpenSearch Dashboards documentation]({{site.url}}{{site.baseurl}}/dashboards/index/).
|
||||
|
|
|
|||
|
|
@ -55,3 +55,7 @@ This problem can occur for two reasons:
|
|||
- You don't have the correct version of `headless-chrome` to match the operating system on which OpenSearch Dashboards is running. Download the correct version [here](https://github.com/opensearch-project/dashboards-reports/releases/tag/chromium-1.12.0.0).
|
||||
|
||||
- You're missing additional dependencies. Install the required dependencies for your operating system from the [additional libraries](https://github.com/opensearch-project/dashboards-reports/blob/main/dashboards-reports/rendering-engine/headless-chrome/README.md#additional-libaries) section.
|
||||
|
||||
### Characters not loading in reports
|
||||
|
||||
You might encounter an issue where UTF-8 encoded characters look fine in your browser, but they don't load in your generated reports because you're missing the required font dependencies. Install the [font dependencies](https://github.com/opensearch-project/dashboards-reports#missing-font-dependencies), and then generate your reports again.
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
message: "🔥 [OpenSearch 1.0 released on July 12th! Get it now!](/downloads.html)"
|
||||
|
|
@ -0,0 +1 @@
|
|||
message: "OpenSearch 1.2.0 is live 🍾 Try the new observability interface, branding customizer, and more!](/downloads.html)"
|
||||
|
|
@ -20,19 +20,22 @@ columns:
|
|||
-
|
||||
title: 'Resources'
|
||||
links:
|
||||
#-
|
||||
#-
|
||||
# title: 'Documentation'
|
||||
# url: 'https://github.com/opensearch/documentation'
|
||||
-
|
||||
-
|
||||
title: FAQ
|
||||
url: '/faq/'
|
||||
-
|
||||
title: Testimonials
|
||||
url: '/testimonials.html'
|
||||
-
|
||||
title: 'Brand Guidelines'
|
||||
url: '/brand.html'
|
||||
-
|
||||
title: 'Trademark Usage Policy'
|
||||
url: '/trademark-usage.html'
|
||||
-
|
||||
-
|
||||
title: OpenSearch Disambiguation
|
||||
url: '/disambiguation.html'
|
||||
-
|
||||
|
|
@ -45,5 +48,5 @@ columns:
|
|||
# title: 'Facebook'
|
||||
# url: 'http://www.facebook.com/opensearch'
|
||||
-
|
||||
title: 'E-mail'
|
||||
url: 'mailto:opensearch@amazon.com'
|
||||
title: 'Connect'
|
||||
url: 'https://opensearch.org/connect.html'
|
||||
|
|
|
|||
|
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"current": "1.2",
|
||||
"past": [
|
||||
"1.1",
|
||||
"1.0"
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
---
|
||||
layout: default
|
||||
title: Settings
|
||||
parent: Index rollups
|
||||
nav_order: 30
|
||||
---
|
||||
|
||||
# Index rollup settings
|
||||
|
||||
We don't recommend changing these settings; the defaults should work well for most use cases.
|
||||
|
||||
All settings are available using the OpenSearch `_cluster/settings` operation. None require a restart, and all can be marked `persistent` or `transient`.
|
||||
|
||||
Setting | Default | Description
|
||||
:--- | :--- | :---
|
||||
`plugins.rollup.search.backoff_millis` | 1000 milliseconds | The backoff time between retries for failed rollup jobs.
|
||||
`plugins.rollup.search.backoff_count` | 5 | How many retries the plugin should attempt for failed rollup jobs.
|
||||
`plugins.rollup.search.search_all_jobs` | false | Whether OpenSearch should return all jobs that match all specified search terms. If disabled, OpenSearch returns just one, as opposed to all, of the jobs that matches the search terms.
|
||||
`plugins.rollup.dashboards.enabled` | true | Whether rollups are enabled in OpenSearch Dashboards.
|
||||
`plugins.rollup.enabled` | true | Whether the rollup plugin is enabled.
|
||||
`plugins.ingest.backoff_millis` | 1000 milliseconds | The backoff time between data ingestions for rollup jobs.
|
||||
`plugins.ingest.backoff_count` | 5 | How many retries the plugin should attempt for failed ingestions.
|
||||
|
|
@ -29,7 +29,7 @@ If you don't have any data in your cluster, you can use the sample flight data w
|
|||
### Step 1: Choose indices
|
||||
|
||||
1. In the **Job name and description** section, specify a name and an optional description for your job.
|
||||
2. In the **Indices** section, select the source and target index. You can either select an existing target index or create a new one by entering a name for your new index. If you want to transform just a subset of your source index, choose **Add Data Filter**, and use the OpenSearch query DSL to specify a subset of your source index. For more information about the OpenSearch query DSL, see [query DSL]({{site.url}}{{site.baseurl}}/opensearch/query-dsl/).
|
||||
2. In the **Indices** section, select the source and target index. You can either select an existing target index or create a new one by entering a name for your new index. If you want to transform just a subset of your source index, choose **Edit data filter**, and use the OpenSearch query DSL to specify a subset of your source index. For more information about the OpenSearch query DSL, see [query DSL]({{site.url}}{{site.baseurl}}/opensearch/query-dsl/).
|
||||
3. Choose **Next**.
|
||||
|
||||
### Step 2: Select fields to transform
|
||||
|
|
@ -51,7 +51,7 @@ On the other hand, aggregations let you perform simple calculations. For example
|
|||
|
||||
You can configure transform jobs to run once or multiple times on a schedule. Transform jobs are enabled by default.
|
||||
|
||||
1. For **transformation execution frequency**, select **Define by fixed interval** and specify a **transform interval**.
|
||||
1. For **transformation execution interval**, specify a transform interval in minutes, hours, or days.
|
||||
2. Under **Advanced**, specify an optional amount for **Pages per execution**. A larger number means more data is processed in each search request, but also uses more memory and causes higher latency. Exceeding allowed memory limits can cause exceptions and errors to occur.
|
||||
3. Choose **Next**.
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
layout: default
|
||||
title: ISM API
|
||||
parent: Index State Management
|
||||
nav_order: 5
|
||||
nav_order: 20
|
||||
---
|
||||
|
||||
# ISM API
|
||||
|
|
|
|||
|
|
@ -31,14 +31,33 @@ To get started, choose **Index Management** in OpenSearch Dashboards.
|
|||
|
||||
A policy is a set of rules that describes how an index should be managed. For information about creating a policy, see [Policies]({{site.url}}{{site.baseurl}}/im-plugin/ism/policies/).
|
||||
|
||||
You can use the visual editor or JSON editor to create policies. Compared to the JSON editor, the visual editor offers a more structured way of defining policies by separating the process into creating error notifications, defining ISM templates, and adding states. We recommend using the visual editor if you want to see pre-defined fields, such as which actions you can assign to a state or under what conditions a state can transition into a destination state.
|
||||
|
||||
#### Visual editor
|
||||
|
||||
1. Choose the **Index Policies** tab.
|
||||
2. Choose **Create policy**.
|
||||
3. In the **Name policy** section, enter a policy ID.
|
||||
4. In the **Define policy** section, enter your policy.
|
||||
5. Choose **Create**.
|
||||
3. Choose **Visual editor**.
|
||||
4. In the **Policy info** section, enter a policy ID and an optional description.
|
||||
5. In the **Error notification** section, set up an optional error notification that gets sent whenever a policy execution fails. For more information, see [Error notifications]({{site.url}}{{site.baseurl}}/im-plugin/ism/policies#error-notifications). If you're using auto rollovers in your policy, we recommend setting up error notifications, which notify you of unexpectedly large indices if rollovers fail.
|
||||
6. In **ISM templates**, enter any ISM template patterns to automatically apply this policy to existing and future indices. For example, if you specify a template of `sample-index*`, the ISM plugin automatically applies this policy to any indices whose names start with `sample-index`.
|
||||
7. In **States**, add any states you want to include in the policy. Each state has [actions]({{site.url}}{{site.baseurl}}/im-plugin/ism/policies/#actions) the plugin executes when the index enters a certain state, and [transitions]({{site.url}}{{site.baseurl}}/im-plugin/ism/policies/#transitions), which have conditions that, when met, transition the index into a destination state. The first state you create in a policy is automatically set as the initial state. Each policy must have at least one state, but actions and transitions are optional.
|
||||
8. Choose **Create**.
|
||||
|
||||
After you create a policy, your next step is to attach this policy to an index or indices.
|
||||
You can set up an `ism_template` in the policy so when you create an index that matches the ISM template pattern, the index will have this policy attached to it:
|
||||
|
||||
#### JSON editor
|
||||
|
||||
1. Choose the **Index Policies** tab.
|
||||
2. Choose **Create policy**.
|
||||
3. Choose **JSON editor**.
|
||||
4. In the **Name policy** section, enter a policy ID.
|
||||
5. In the **Define policy** section, enter your policy.
|
||||
6. Choose **Create**.
|
||||
|
||||
After you create a policy, your next step is to attach it to an index or indices.
|
||||
You can set up an `ism_template` in the policy so when an index that matches the ISM template pattern is created, the plugin automatically attaches the policy to the index.
|
||||
|
||||
The following example demonstrates how to create a policy that automatically gets attached to all indices whose names start with `index_name-`.
|
||||
|
||||
```json
|
||||
PUT _plugins/_ism/policies/policy_id
|
||||
|
|
@ -57,7 +76,7 @@ PUT _plugins/_ism/policies/policy_id
|
|||
|
||||
If you have more than one template that matches an index pattern, ISM uses the priority value to determine which template to apply.
|
||||
|
||||
For an example ISM template policy, see [Sample policy with ISM template]({{site.url}}{{site.baseurl}}/im-plugin/ism/policies#sample-policy-with-ism-template).
|
||||
For an example ISM template policy, see [Sample policy with ISM template for auto rollover]({{site.url}}{{site.baseurl}}/im-plugin/ism/policies#sample-policy-with-ism-template-for-auto-rollover).
|
||||
|
||||
Older versions of the plugin include the `policy_id` in an index template, so when an index is created that matches the index template pattern, the index will have the policy attached to it:
|
||||
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ Field | Description | Type | Required | Read Only
|
|||
|
||||
---
|
||||
|
||||
## States
|
||||
## States
|
||||
|
||||
A state is the description of the status that the managed index is currently in. A managed index can be in only one state at a time. Each state has associated actions that are executed sequentially on entering a state and transitions that are checked after all the actions have been completed.
|
||||
|
||||
|
|
@ -71,7 +71,7 @@ The `retry` operation has the following parameters:
|
|||
Parameter | Description | Type | Required | Default
|
||||
:--- | :--- |:--- |:--- |
|
||||
`count` | The number of retry counts. | `number` | Yes | -
|
||||
`backoff` | The backoff policy type to use when retrying. | `string` | No | Exponential
|
||||
`backoff` | The backoff policy type to use when retrying. Valid values are Exponential, Constant, and Linear. | `string` | No | Exponential
|
||||
`delay` | The time to wait between retries. Accepts time units for minutes, hours, and days. | `time unit` | No | 1 minute
|
||||
|
||||
The following example action has a timeout period of one hour. The policy retries this action three times with an exponential backoff policy, with a delay of 10 minutes between each retry:
|
||||
|
|
@ -314,13 +314,13 @@ The `snapshot` operation has the following parameters:
|
|||
Parameter | Description | Type | Required | Default
|
||||
:--- | :--- |:--- |:--- |
|
||||
`repository` | The repository name that you register through the native snapshot API operations. | `string` | Yes | -
|
||||
`snapshot` | The name of the snapshot. | `string` | Yes | -
|
||||
`snapshot` | The name of the snapshot. Accepts strings and the Mustache variables `{{ctx.index}}` and `{{ctx.indexUuid}}`. If the Mustache variables are invalid, then the snapshot name defaults to the index's name. | `string` or Mustache template | Yes | -
|
||||
|
||||
```json
|
||||
{
|
||||
"snapshot": {
|
||||
"repository": "my_backup",
|
||||
"snapshot": "my_snapshot"
|
||||
"snapshot": "{{ctx.indexUuid}}"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
|
@ -517,10 +517,12 @@ The destination system **must** return a response otherwise the `error_notificat
|
|||
|
||||
You can use the same options for `ctx` variables as the [notification](#notification) operation.
|
||||
|
||||
## Sample policy with ISM template
|
||||
## Sample policy with ISM template for auto rollover
|
||||
|
||||
The following sample template policy is for a rollover use case.
|
||||
|
||||
If you want to skip rollovers for an index, use the [update cluster settings API]({{site.url}}{{site.baseurl}}/opensearch/configuration/#update-cluster-settings-using-the-api) to set `index.plugins.index_state_management.rollover_skip` to true.
|
||||
|
||||
1. Create a policy with an `ism_template` field:
|
||||
|
||||
```json
|
||||
|
|
@ -558,9 +560,11 @@ The following sample template policy is for a rollover use case.
|
|||
PUT _index_template/ism_rollover
|
||||
{
|
||||
"index_patterns": ["log*"],
|
||||
"settings": {
|
||||
"template": {
|
||||
"settings": {
|
||||
"plugins.index_state_management.rollover_alias": "log"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -586,6 +590,12 @@ The following sample template policy is for a rollover use case.
|
|||
}
|
||||
```
|
||||
|
||||
5. Verify if the policy is attached to the `log-000001` index:
|
||||
|
||||
```json
|
||||
GET _plugins/_ism/explain/log-000001?pretty
|
||||
```
|
||||
|
||||
## Example policy
|
||||
|
||||
The following example policy implements a `hot`, `warm`, and `delete` workflow. You can use this policy as a template to prioritize resources to your indices based on their levels of activity.
|
||||
|
|
@ -659,6 +669,6 @@ After 30 days, the policy moves this index into a `delete` state. The service se
|
|||
}
|
||||
```
|
||||
|
||||
This diagram shows the `states`, `transitions`, and `actions` of the above policy as a finite-state machine. For more information about finite-state machines, see [Wikipedia](https://en.wikipedia.org/wiki/Finite-state_machine).
|
||||
This diagram shows the `states`, `transitions`, and `actions` of the above policy as a finite-state machine. For more information about finite-state machines, see [Wikipedia](https://en.wikipedia.org/wiki/Finite-state_machine).
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ Setting | Default | Description
|
|||
:--- | :--- | :---
|
||||
`plugins.index_state_management.enabled` | True | Specifies whether ISM is enabled or not.
|
||||
`plugins.index_state_management.job_interval` | 5 minutes | The interval at which the managed index jobs are run.
|
||||
`plugins.index_state_management.jitter` | 0.6 | A randomized delay that is added to a job's base run time to prevent a surge of activity from all indices at the same time. A value of 0.6 means a delay of 0-60% of a job interval is added to the base interval. For example, if you have a base interval time of 30 minutes, a value of 0.6 means an amount anywhere between 0 to 18 minutes gets added to your job interval. Maximum is 1, which means an additional interval time of 100%. This maximum cannot exceed `plugins.jobscheduler.jitter_limit`, which also has a default of 0.6. For example, if `plugins.index_state_management.jitter` is set to 0.8, ISM uses `plugins.jobscheduler.jitter_limit` of 0.6 instead.
|
||||
`plugins.index_state_management.coordinator.sweep_period` | 10 minutes | How often the routine background sweep is run.
|
||||
`plugins.index_state_management.coordinator.backoff_millis` | 50 milliseconds | The backoff time between retries for failures in the `ManagedIndexCoordinator` (such as when we update managed indices).
|
||||
`plugins.index_state_management.coordinator.backoff_count` | 2 | The count of retries for failures in the `ManagedIndexCoordinator`.
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: default
|
||||
title: Refresh search analyzer
|
||||
nav_order: 40
|
||||
nav_order: 50
|
||||
has_children: false
|
||||
redirect_from: /im-plugin/refresh-analyzer/
|
||||
has_toc: false
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
---
|
||||
layout: default
|
||||
title: Index management security
|
||||
nav_order: 40
|
||||
has_children: false
|
||||
---
|
||||
|
||||
# Index management security
|
||||
|
||||
Using the security plugin with index management lets you limit non-admin users to certain actions. For example, you might want to set up your security such that a group of users can only read ISM policies, while others can create, delete, or change policies.
|
||||
|
||||
All index management data are protected as system indices, and only a super admin or an admin with a Transport Layer Security (TLS) certificate can access system indices. For more information, see [System indices]({{site.url}}{{site.baseurl}}/security-plugin/configuration/system-indices).
|
||||
|
||||
## Basic permissions
|
||||
|
||||
The security plugin comes with one role that offers full access to index management: `index_management_full_access`. For a description of the role's permissions, see [Predefined roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/users-roles#predefined-roles).
|
||||
|
||||
With security enabled, users not only need the correct index management permissions, but they also need permissions to execute actions to involved indices. For example, if a user wants to use the REST API to attach a policy that executes a rollup job to an index named `system-logs`, they would need the permissions to attach a policy and execute a rollup job, as well as access to `system-logs`.
|
||||
|
||||
Finally, with the exceptions of Create Policy, Get Policy, and Delete Policy, users also need the `indices:admin/opensearch/ism/managedindex` permission to execute [ISM APIs]({{site.url}}{{site.baseurl}}/im-plugin/ism/api).
|
||||
|
||||
## (Advanced) Limit access by backend role
|
||||
|
||||
You can use backend roles to configure fine-grained access to index management policies and actions. For example, users of different departments in an organization might view different policies depending on what roles and permissions they are assigned.
|
||||
|
||||
First, ensure your users have the appropriate [backend roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/index/). Backend roles usually come from an [LDAP server]({{site.url}}{{site.baseurl}}/security-plugin/configuration/ldap/) or [SAML provider]({{site.url}}{{site.baseurl}}/security-plugin/configuration/saml/). However, if you use the internal user database, you can use the REST API to [add them manually]({{site.url}}{{site.baseurl}}/security-plugin/access-control/api#create-user).
|
||||
|
||||
Use the REST API to enable the following setting:
|
||||
|
||||
```json
|
||||
PUT _cluster/settings
|
||||
{
|
||||
"transient": {
|
||||
"plugins.index_management.filter_by_backend_roles": "true"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
With security enabled, only users who share at least one backend role can see and execute the policies and actions relevant to their roles.
|
||||
|
||||
For example, consider a scenario with three users: `John` and `Jill`, who have the backend role `helpdesk_staff`, and `Jane`, who has the backend role `phone_operator`. `John` wants to create a policy that performs a rollup job on an index named `airline_data`, so `John` would need a backend role that has permissions to access that index, create relevant policies, and execute relevant actions, and `Jill` would be able to access the same index, policy, and job. However, `Jane` cannot access or edit those resources or actions.
|
||||
|
|
@ -3,10 +3,10 @@
|
|||
<div class="subfooter">
|
||||
<div class="container">
|
||||
<h1 class="visuallyhidden">OpenSearch Links</h1>
|
||||
|
||||
|
||||
{% for column in site.data.footer.columns %}
|
||||
<div class="col {% if forloop.index > 2 %}last-child{% endif %}">
|
||||
|
||||
|
||||
<h2>{{ column.title }}</h2>
|
||||
<ul>
|
||||
{% for link in column.links %}
|
||||
|
|
@ -27,17 +27,17 @@
|
|||
<path d="M48.0814 38C50.2572 34.4505 52.3615 29.7178 51.9475 23.0921C51.0899 9.36725 38.6589 -1.04463 26.9206 0.0837327C22.3253 0.525465 17.6068 4.2712 18.026 10.9805C18.2082 13.8961 19.6352 15.6169 21.9544 16.9399C24.1618 18.1992 26.9978 18.9969 30.2128 19.9011C34.0962 20.9934 38.6009 22.2203 42.063 24.7717C46.2125 27.8295 49.0491 31.3743 48.0814 38Z" fill="currentColor" />
|
||||
<path d="M3.91861 14C1.74276 17.5495 -0.361506 22.2822 0.0524931 28.9079C0.910072 42.6327 13.3411 53.0446 25.0794 51.9163C29.6747 51.4745 34.3932 47.7288 33.974 41.0195C33.7918 38.1039 32.3647 36.3831 30.0456 35.0601C27.8382 33.8008 25.0022 33.0031 21.7872 32.0989C17.9038 31.0066 13.3991 29.7797 9.93694 27.2283C5.78746 24.1704 2.95092 20.6257 3.91861 14Z" fill="currentColor" />
|
||||
</svg></a>
|
||||
|
||||
|
||||
<p class="copyright">© {{ 'now' | date: "%Y" }}
|
||||
<a href="https://aws.amazon.com/"> Amazon Web Services</a> and individual contributors. OpenSearch is a
|
||||
<a href="/trademark-usage.html">registered trademark</a> of Amazon Web Services.</a> <br /><br />
|
||||
<a href="https://opensearch.org/"> OpenSearch</a> contributors. OpenSearch is a
|
||||
<a href="/trademark-usage.html">registered trademark</a> of <a href="https://aws.amazon.com/"> Amazon Web Services</a>. <br /><br />
|
||||
|
||||
© 2005-2021
|
||||
<a href="https://www.djangoproject.com/foundation/"> Django Software
|
||||
Foundation</a> and individual contributors. Django is a
|
||||
<a href="https://www.djangoproject.com/trademarks/">registered
|
||||
trademark</a> of the Django Software Foundation.<br />
|
||||
This website was forked from the BSD-licensed <a href="https://github.com/django/djangoproject.com/">djangoproject.com</a> originally designed by <a href="https://www.threespot.com">Threespot</a> <span class="ampersand">&</span> <a href="http://andrevv.com/">andrevv</a>.<br /> We ♡ Django and the Django community. If you need a <a href="https://www.djangoproject.com/">high-level Python framework</a>, check it out.
|
||||
This website was forked from the BSD-licensed <a href="https://github.com/django/djangoproject.com/">djangoproject.com</a> originally designed by <a href="https://www.threespot.com">Threespot</a> <span class="ampersand">&</span> <a href="http://andrevv.com/">andrevv</a>.<br /> We ♡ Django and the Django community. If you need a <a href="https://www.djangoproject.com/">high-level Python framework</a>, check it out.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -6,3 +6,9 @@
|
|||
<script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
|
||||
<script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3.0.1/es5/tex-mml-chtml.js"></script>
|
||||
{% endif %}
|
||||
|
||||
{% if jekyll.environment == "development" %}
|
||||
<script src="{{ '/assets/js/version-selector.js' | relative_url }}"></script>
|
||||
{% else %}
|
||||
<script src="{{ '/docs/latest/assets/js/version-selector.js' }}"></script>
|
||||
{% endif %}
|
||||
|
|
|
|||
|
|
@ -57,6 +57,10 @@ layout: table_wrappers
|
|||
</a>
|
||||
</div>
|
||||
<nav role="navigation" aria-label="Main" id="site-nav" class="site-nav">
|
||||
{% assign past_versions = site.data.versions.past | join: ";" %}
|
||||
<div class="version-wrapper">
|
||||
<version-selector selected="{{ site.data.versions.current }}"></version-selector>
|
||||
</div>
|
||||
{% assign pages_top_size = site.html_pages
|
||||
| where_exp:"item", "item.title != nil"
|
||||
| where_exp:"item", "item.parent == nil"
|
||||
|
|
@ -101,13 +105,24 @@ layout: table_wrappers
|
|||
<div class="copy-banner">
|
||||
<div class="container">
|
||||
<h1><a href="#">Documentation</a></h1>
|
||||
{% if site.search_enabled != false %}
|
||||
{% if site.search_enabled != false or site.use_custom_search == true %}
|
||||
<div class="search">
|
||||
<div class="search-input-wrap">
|
||||
<input type="text" id="search-input" class="search-input" tabindex="0" placeholder="Search..." aria-label="Search {{ site.title }}" autocomplete="off">
|
||||
{% assign docs_version = site.data.versions.current %}
|
||||
{% if site.baseurl == '/docs/latest' %}
|
||||
{% assign docs_version = "latest" %}
|
||||
{% endif %}
|
||||
<input type="text" id="search-input" class="search-input"
|
||||
tabindex="0" placeholder="Search..." aria-label="Search {{ site.title }}"
|
||||
data-docs-version="{{ docs_version }}" autocomplete="off">
|
||||
<div class="search-spinner"><i></i></div>
|
||||
<label for="search-input" class="search-label"><svg viewBox="0 0 24 24" class="search-icon"><use xlink:href="#svg-search"></use></svg></label>
|
||||
</div>
|
||||
{% if site.search_enabled != false %}
|
||||
<div id="search-results" class="search-results"></div>
|
||||
{% elsif site.use_custom_search == true %}
|
||||
<div id="search-results" class="search-results custom-search-results"></div>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
|
@ -209,5 +224,8 @@ layout: table_wrappers
|
|||
</script>
|
||||
{% endif %}
|
||||
<script src="{{ '/assets/js/header-nav.js' | relative_url }}"></script>
|
||||
{% if site.search_enabled == false and site.use_custom_search == true %}
|
||||
<script src="{{ '/assets/js/search.js' | relative_url }}"></script>
|
||||
{% endif %}
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -17,69 +17,92 @@ Anomaly detection automatically detects anomalies in your OpenSearch data in ne
|
|||
|
||||
You can pair the anomaly detection plugin with the [alerting plugin]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/) to notify you as soon as an anomaly is detected.
|
||||
|
||||
To use the anomaly detection plugin, your computer needs to have more than one CPU core.
|
||||
{: .note }
|
||||
|
||||
## Get started with Anomaly Detection
|
||||
|
||||
To get started, choose **Anomaly Detection** in OpenSearch Dashboards.
|
||||
To first test with sample streaming data, choose **Sample Detectors** and try out one of the preconfigured detectors.
|
||||
To first test with sample streaming data, you can try out one of the preconfigured detectors with one of the sample datasets.
|
||||
|
||||
### Step 1: Create a detector
|
||||
## Step 1: Define a detector
|
||||
|
||||
A detector is an individual anomaly detection task. You can create multiple detectors, and all the detectors can run simultaneously, with each analyzing data from different sources.
|
||||
A detector is an individual anomaly detection task. You can define multiple detectors, and all the detectors can run simultaneously, with each analyzing data from different sources.
|
||||
|
||||
1. Choose **Create Detector**.
|
||||
1. Enter a name and brief description. Make sure the name is unique and descriptive enough to help you to identify the purpose of the detector.
|
||||
1. For **Data source**, choose the index you want to use as the data source. You can optionally use index patterns to choose multiple indices.
|
||||
1. Select the **Timestamp field** in your index.
|
||||
1. (Optional) For **Data filter**, filter the index you chose as the data source. From the **Filter type** menu, choose **Visual filter**, and then design your filter query by selecting **Fields**, **Operator**, and **Value**, or choose **Custom Expression** and add your own JSON filter query.
|
||||
1. For **Detector operation settings**, define the **Detector interval**, which is the time interval at which the detector collects data.
|
||||
- The detector aggregates the data in this interval, then feeds the aggregated result into the anomaly detection model.
|
||||
The shorter you set this interval, the fewer data points the detector aggregates.
|
||||
The anomaly detection model uses a shingling process, a technique that uses consecutive data points to create a sample for the model. This process needs a certain number of aggregated data points from contiguous intervals.
|
||||
- We recommend setting the detector interval based on your actual data. If it's too long it might delay the results, and if it's too short it might miss some data. It also won't have a sufficient number of consecutive data points for the shingle process.
|
||||
1. (Optional) To add extra processing time for data collection, specify a **Window delay** value. This value tells the detector that the data is not ingested into OpenSearch in real time but with a certain delay.
|
||||
Set the window delay to shift the detector interval to account for this delay.
|
||||
- For example, say the detector interval is 10 minutes and data is ingested into your cluster with a general delay of 1 minute.
|
||||
Assume the detector runs at 2:00. The detector attempts to get the last 10 minutes of data from 1:50 to 2:00, but because of the 1-minute delay, it only gets 9 minutes of data and misses the data from 1:59 to 2:00.
|
||||
Setting the window delay to 1 minute shifts the interval window to 1:49 - 1:59, so the detector accounts for all 10 minutes of the detector interval time.
|
||||
1. Choose **Create**.
|
||||
1. Choose **Create detector**.
|
||||
1. Add in the detector details.
|
||||
- Enter a name and brief description. Make sure the name is unique and descriptive enough to help you to identify the purpose of the detector.
|
||||
1. Specify the data source.
|
||||
- For **Data source**, choose the index you want to use as the data source. You can optionally use index patterns to choose multiple indices.
|
||||
- (Optional) For **Data filter**, filter the index you chose as the data source. From the **Data filter** menu, choose **Add data filter**, and then design your filter query by selecting **Field**, **Operator**, and **Value**, or choose **Use query DSL** and add your own JSON filter query.
|
||||
1. Specify a timestamp.
|
||||
- Select the **Timestamp field** in your index.
|
||||
1. Define operation settings.
|
||||
- For **Operation settings**, define the **Detector interval**, which is the time interval at which the detector collects data.
|
||||
- The detector aggregates the data in this interval, then feeds the aggregated result into the anomaly detection model.
|
||||
The shorter you set this interval, the fewer data points the detector aggregates.
|
||||
The anomaly detection model uses a shingling process, a technique that uses consecutive data points to create a sample for the model. This process needs a certain number of aggregated data points from contiguous intervals.
|
||||
|
||||
After you create the detector, the next step is to add features to it.
|
||||
- We recommend setting the detector interval based on your actual data. If it's too long it might delay the results, and if it's too short it might miss some data. It also won't have a sufficient number of consecutive data points for the shingle process.
|
||||
|
||||
### Step 2: Add features to your detector
|
||||
- (Optional) To add extra processing time for data collection, specify a **Window delay** value.
|
||||
- This value tells the detector that the data is not ingested into OpenSearch in real time but with a certain delay. Set the window delay to shift the detector interval to account for this delay.
|
||||
- For example, say the detector interval is 10 minutes and data is ingested into your cluster with a general delay of 1 minute. Assume the detector runs at 2:00. The detector attempts to get the last 10 minutes of data from 1:50 to 2:00, but because of the 1-minute delay, it only gets 9 minutes of data and misses the data from 1:59 to 2:00. Setting the window delay to 1 minute shifts the interval window to 1:49 - 1:59, so the detector accounts for all 10 minutes of the detector interval time.
|
||||
1. Specify custom result index.
|
||||
- If you want to store the anomaly detection results in your own index, choose **Enable custom result index** and specify the custom index to store the result. The anomaly detection plugin adds an `opensearch-ad-plugin-result-` prefix to the index name that you input. For example, if you input `abc` as the result index name, the final index name is `opensearch-ad-plugin-result-abc`.
|
||||
|
||||
You can use the dash “-” sign to separate the namespace to manage custom result index permissions. For example, if you use `opensearch-ad-plugin-result-financial-us-group1` as the result index, you can create a permission role based on the pattern `opensearch-ad-plugin-result-financial-us-*` to represent the "financial" department at a granular level for the "us" area.
|
||||
{: .note }
|
||||
|
||||
- If the custom index you specify doesn’t already exist, the anomaly detection plugin creates this index when you create the detector and start your real-time or historical analysis.
|
||||
- If the custom index already exists, the plugin checks if the index mapping of the custom index matches the anomaly result file. You need to make sure the custom index has valid mapping as shown here: [anomaly-results.json](https://github.com/opensearch-project/anomaly-detection/blob/main/src/main/resources/mappings/anomaly-results.json).
|
||||
- To use the custom result index option, you need the following permissions:
|
||||
- `indices:admin/create` - If the custom index already exists, you don't need this.
|
||||
- `indices:data/write/index` - You need the `write` permission for the anomaly detection plugin to write results into the custom index for a single-entity detector.
|
||||
- `indices:data/read/search` - You need the `search` permission because the anomaly detection plugin needs to search custom result indices to show results on the anomaly detection UI.
|
||||
- `indices:data/write/delete` - Because the detector might generate a large number of anomaly results, you need the `delete` permission to delete old data and save disk space.
|
||||
- `indices:data/write/bulk*` - You need the `bulk*` permission because the anomaly detection plugin uses the bulk API to write results into the custom index.
|
||||
- Managing the custom result index:
|
||||
- The anomaly detection dashboard queries all detectors’ results from all custom result indices. Having too many custom result indices might impact the performance of the anomaly detection plugin.
|
||||
- You can use [Index State Management]({{site.url}}{{site.baseurl}}/im-plugin/ism/index/) to rollover old result indices. You can also manually delete or archive any old result indices. We recommend reusing a custom result index for multiple detectors.
|
||||
1. Choose **Next**.
|
||||
|
||||
After you define the detector, the next step is to configure the model.
|
||||
|
||||
## Step 2: Configure the model
|
||||
|
||||
#### Add features to your detector
|
||||
|
||||
A feature is the field in your index that you want to check for anomalies. A detector can discover anomalies across one or more features. You must choose an aggregation method for each feature: `average()`, `count()`, `sum()`, `min()`, or `max()`. The aggregation method determines what constitutes an anomaly.
|
||||
|
||||
For example, if you choose `min()`, the detector focuses on finding anomalies based on the minimum values of your feature. If you choose `average()`, the detector finds anomalies based on the average values of your feature.
|
||||
|
||||
A multi-feature model correlates anomalies across all its features. The [curse of dimensionality](https://en.wikipedia.org/wiki/Curse_of_dimensionality) makes it less likely for multi-feature models to identify smaller anomalies as compared to a single-feature model. Adding more features might negatively impact the [precision and recall](https://en.wikipedia.org/wiki/Precision_and_recall) of a model. A higher proportion of noise in your data might further amplify this negative impact. Selecting the optimal feature set is usually an iterative process. We recommend experimenting with a historical detector with different feature sets and checking the precision before moving on to real-time detectors. By default, the maximum number of features for a detector is 5. You can adjust this limit with the `plugins.anomaly_detection.max_anomaly_features` setting.
|
||||
A multi-feature model correlates anomalies across all its features. The [curse of dimensionality](https://en.wikipedia.org/wiki/Curse_of_dimensionality) makes it less likely for multi-feature models to identify smaller anomalies as compared to a single-feature model. Adding more features might negatively impact the [precision and recall](https://en.wikipedia.org/wiki/Precision_and_recall) of a model. A higher proportion of noise in your data might further amplify this negative impact. Selecting the optimal feature set is usually an iterative process. By default, the maximum number of features for a detector is 5. You can adjust this limit with the `plugins.anomaly_detection.max_anomaly_features` setting.
|
||||
{: .note }
|
||||
|
||||
1. On the **Model configuration** page, enter the **Feature name**.
|
||||
1. For **Find anomalies based on**, choose the method to find anomalies. For **Field Value** menu, choose the **field** and the **aggregation method**. Or choose **Custom expression**, and add your own JSON aggregation query.
|
||||
1. On the **Configure Model** page, enter the **Feature name** and check **Enable feature**.
|
||||
1. For **Find anomalies based on**, choose the method to find anomalies. For **Field Value**, choose the **aggregation method**. Or choose **Custom expression**, and add your own JSON aggregation query.
|
||||
1. Select a field.
|
||||
|
||||
#### (Optional) Set a category field for high cardinality
|
||||
#### (Optional) Set category fields for high cardinality
|
||||
|
||||
You can categorize anomalies based on a keyword or IP field type.
|
||||
|
||||
The category field categorizes or slices the source time series with a dimension like IP addresses, product IDs, country codes, and so on. This helps to see a granular view of anomalies within each entity of the category field to isolate and debug issues.
|
||||
|
||||
To set a category field, choose **Enable a category field** and select a field.
|
||||
To set a category field, choose **Enable a category field** and select a field. You can’t change the category fields after you create the detector.
|
||||
|
||||
Only a certain number of unique entities are supported in the category field. Use the following equation to calculate the recommended total number of entities supported in a cluster:
|
||||
|
||||
```
|
||||
(data nodes * heap size * anomaly detection maximum memory percentage) / (entity size of a detector)
|
||||
(data nodes * heap size * anomaly detection maximum memory percentage) / (entity model size of a detector)
|
||||
```
|
||||
|
||||
To get the entity model size of a detector, use the [profile detector API]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/api/#profile-detector). You can adjust the maximum memory percentage with the `plugins.anomaly_detection.model_max_size_percent` setting.
|
||||
|
||||
This formula provides a good starting point, but make sure to test with a representative workload.
|
||||
{: .note }
|
||||
|
||||
For example, for a cluster with 3 data nodes, each with 8G of JVM heap size, a maximum memory percentage of 10% (default), and the entity size of the detector as 1MB: the total number of unique entities supported is (8.096 * 10^9 * 0.1 / 1M ) * 3 = 2429.
|
||||
For example, for a cluster with three data nodes, each with 8 GB of JVM heap size, a maximum memory percentage of 10% (default), and the entity model size of the detector as 1MB: the total number of unique entities supported is (8.096 * 10^9 * 0.1 / 1 MB ) * 3 = 2429.
|
||||
|
||||
#### Set a shingle size
|
||||
If the actual total number of unique entities higher than this number that you calculate (in this case: 2429), the anomaly detector makes its best effort to model the extra entities. The detector prioritizes entities that occur more often and are more recent.
|
||||
|
||||
#### (Advanced settings) Set a shingle size
|
||||
|
||||
Set the number of aggregation intervals from your data stream to consider in a detection window. It’s best to choose this value based on your actual data to see which one leads to the best results for your use case.
|
||||
|
||||
|
|
@ -92,12 +115,32 @@ For sample previews, the anomaly detection plugin selects a small number of data
|
|||
|
||||
Examine the sample preview and use it to fine-tune your feature configurations (for example, enable or disable features) to get more accurate results.
|
||||
|
||||
1. Choose **Save and start detector**.
|
||||
1. Choose between automatically starting the detector (recommended) or manually starting the detector at a later time.
|
||||
1. Choose **Preview sample anomalies**.
|
||||
- If you don't see any sample anomaly result, check the detector interval and make sure you have more than 400 data points for some entities during the preview date range.
|
||||
1. Choose **Next**.
|
||||
|
||||
### Step 3: Observe the results
|
||||
## Step 3: Set up detector jobs
|
||||
|
||||
Choose the **Anomaly results** tab. You need to wait for some time to see the anomaly results. If the detector interval is 10 minutes, the detector might take more than an hour to start, as it's waiting for sufficient data to generate anomalies.
|
||||
To start a real-time detector to find anomalies in your data in near real-time, check **Start real-time detector automatically (recommended)**.
|
||||
|
||||
Alternatively, if you want to perform historical analysis and find patterns in long historical data windows (weeks or months), check **Run historical analysis detection** and select a date range (at least 128 detection intervals).
|
||||
|
||||
Analyzing historical data helps you get familiar with the anomaly detection plugin. You can also evaluate the performance of a detector with historical data to further fine-tune it.
|
||||
|
||||
We recommend experimenting with historical analysis with different feature sets and checking the precision before moving on to real-time detectors.
|
||||
|
||||
## Step 4: Review and create
|
||||
|
||||
Review your detector settings and model configurations to make sure that they're valid and then select **Create detector**.
|
||||
|
||||
|
||||
|
||||
If you see any validation errors, edit the settings to fix the errors and then return back to this page.
|
||||
{: .note }
|
||||
|
||||
## Step 5: Observe the results
|
||||
|
||||
Choose the **Real-time results** or **Historical analysis** tab. For real-time results, you need to wait for some time to see the anomaly results. If the detector interval is 10 minutes, the detector might take more than an hour to start, as it's waiting for sufficient data to generate anomalies.
|
||||
|
||||
A shorter interval means the model passes the shingle process more quickly and starts to generate the anomaly results sooner.
|
||||
Use the [profile detector]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/api#profile-detector) operation to make sure you have sufficient data points.
|
||||
|
|
@ -106,12 +149,12 @@ If you see the detector pending in "initialization" for longer than a day, aggre
|
|||
|
||||
|
||||
|
||||
Analize anomalies with the following visualizations:
|
||||
Analyze anomalies with the following visualizations:
|
||||
|
||||
- **Live anomalies** - displays live anomaly results for the last 60 intervals. For example, if the interval is 10, it shows results for the last 600 minutes. The chart refreshes every 30 seconds.
|
||||
- **Anomaly history** - plots the anomaly grade with the corresponding measure of confidence.
|
||||
- **Feature breakdown** - plots the features based on the aggregation method. You can vary the date-time range of the detector.
|
||||
- **Anomaly history** (for historical analysis) / **Anomaly overview** (for real-time results) - plots the anomaly grade with the corresponding measure of confidence.
|
||||
- **Anomaly occurrence** - shows the `Start time`, `End time`, `Data confidence`, and `Anomaly grade` for each detected anomaly.
|
||||
- **Feature breakdown** - plots the features based on the aggregation method. You can vary the date-time range of the detector.
|
||||
|
||||
`Anomaly grade` is a number between 0 and 1 that indicates how anomalous a data point is. An anomaly grade of 0 represents “not an anomaly,” and a non-zero value represents the relative severity of the anomaly.
|
||||
|
||||
|
|
@ -119,47 +162,32 @@ Analize anomalies with the following visualizations:
|
|||
|
||||
If you set the category field, you see an additional **Heat map** chart. The heat map correlates results for anomalous entities. This chart is empty until you select an anomalous entity. You also see the anomaly and feature line chart for the time period of the anomaly (`anomaly_grade` > 0).
|
||||
|
||||
Choose a filled rectangle to see a more detailed view of the anomaly.
|
||||
Choose and drag over the anomaly line chart to zoom in and see a more detailed view of an anomaly.
|
||||
{: .note }
|
||||
|
||||
### Step 4: Set up alerts
|
||||
If you have set multiple category fields, you can select a subset of fields to filter and sort the fields by. Selecting a subset of fields lets you see the top values of one field that share a common value with another field.
|
||||
|
||||
Choose **Set up alerts** and configure a monitor to notify you when anomalies are detected. For steps to create a monitor and set up notifications based on your anomaly detector, see [Monitors]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/monitors/).
|
||||
For example, if you have a detector with the category fields `ip` and `endpoint`, you can select `endpoint` in the **View by** dropdown menu. Then, select a specific cell to overlay the top 20 values of `ip` on the charts. The anomaly detection plugin selects the top `ip` by default. You can see a maximum of 5 individual time-series values at the same time.
|
||||
|
||||
|
||||
|
||||
## Step 6: Set up alerts
|
||||
|
||||
Under **Real-time results**, choose **Set up alerts** and configure a monitor to notify you when anomalies are detected. For steps to create a monitor and set up notifications based on your anomaly detector, see [Monitors]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/monitors/).
|
||||
|
||||
If you stop or delete a detector, make sure to delete any monitors associated with it.
|
||||
|
||||
### Step 5: Adjust the model
|
||||
## Step 7: Adjust the model
|
||||
|
||||
To see all the configuration settings for a detector, choose the **Detector configuration** tab.
|
||||
|
||||
1. To make any changes to the detector configuration, or fine tune the time interval to minimize any false positives, go to the **Detector configuration** section and choose **Edit**.
|
||||
- You need to stop the detector to change its configuration. Confirm that you want to stop the detector and proceed.
|
||||
- You need to stop real-time and historical analysis to change its configuration. Confirm that you want to stop the detector and proceed.
|
||||
1. To enable or disable features, in the **Features** section, choose **Edit** and adjust the feature settings as needed. After you make your changes, choose **Save and start detector**.
|
||||
- Choose between automatically starting the detector (recommended) or manually starting the detector at a later time.
|
||||
|
||||
### Step 6: Analyze historical data
|
||||
## Step 8: Manage your detectors
|
||||
|
||||
Analyzing historical data helps you get familiar with the anomaly detection plugin. You can also evaluate the performance of a detector with historical data to further fine-tune it.
|
||||
To start, stop, or delete a detector, go to the **Detectors** page.
|
||||
|
||||
To use a historical detector, you need to specify a date range that has data present in at least 1,000 detection intervals.
|
||||
{: .note }
|
||||
|
||||
1. Choose **Historical detectors** and **Create historical detector**.
|
||||
1. Enter the **Name** of the detector and a brief **Description**.
|
||||
1. For **Data source**, choose the index to use as the data source. You can optionally use index patterns to choose multiple indices.
|
||||
1. For **Time range**, select a time range for historical analysis.
|
||||
1. For **Detector settings**, choose to use the settings of an existing detector. Or choose the **Timestamp field** in your index, add individual features to the detector, and set the detector interval.
|
||||
1. (Optional) Choose to run the historical detector automatically after creating it.
|
||||
1. Choose **Create**.
|
||||
- You can stop the historical detector even before it completes.
|
||||
|
||||
### Step 7: Manage your detectors
|
||||
|
||||
To change or delete a detector, go to the **Detector details** page.
|
||||
|
||||
1. To make changes to your detector, choose the detector name.
|
||||
1. Choose **Actions** and **Edit detector**.
|
||||
- You need to stop the detector to change its configuration. Confirm that you want to stop the detector and proceed.
|
||||
1. Make your changes and choose **Save changes**.
|
||||
|
||||
To delete your detector, choose **Actions** and **Delete detector**. In the pop-up box, type `delete` to confirm and choose **Delete**.
|
||||
1. Choose the detector name.
|
||||
2. Choose **Actions** and select **Start real-time detectors**, **Stop real-time detectors**, or **Delete detectors**.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,319 @@
|
|||
---
|
||||
layout: default
|
||||
title: Anomaly result mapping
|
||||
parent: Anomaly detection
|
||||
nav_order: 6
|
||||
---
|
||||
|
||||
# Anomaly result mapping
|
||||
|
||||
If you enabled custom result index, the anomaly detection plugin stores the results in your own index.
|
||||
|
||||
If the anomaly detector doesn’t detect an anomaly, the result has the following format:
|
||||
|
||||
```json
|
||||
{
|
||||
"detector_id": "kzcZ43wBgEQAbjDnhzGF",
|
||||
"schema_version": 5,
|
||||
"data_start_time": 1635898161367,
|
||||
"data_end_time": 1635898221367,
|
||||
"feature_data": [
|
||||
{
|
||||
"feature_id": "processing_bytes_max",
|
||||
"feature_name": "processing bytes max",
|
||||
"data": 2322
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_avg",
|
||||
"feature_name": "processing bytes avg",
|
||||
"data": 1718.6666666666667
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_min",
|
||||
"feature_name": "processing bytes min",
|
||||
"data": 1375
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_sum",
|
||||
"feature_name": "processing bytes sum",
|
||||
"data": 5156
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_time_max",
|
||||
"feature_name": "processing time max",
|
||||
"data": 31198
|
||||
}
|
||||
],
|
||||
"execution_start_time": 1635898231577,
|
||||
"execution_end_time": 1635898231622,
|
||||
"anomaly_score": 1.8124904404395776,
|
||||
"anomaly_grade": 0,
|
||||
"confidence": 0.9802940756605277,
|
||||
"entity": [
|
||||
{
|
||||
"name": "process_name",
|
||||
"value": "process_3"
|
||||
}
|
||||
],
|
||||
"model_id": "kzcZ43wBgEQAbjDnhzGF_entity_process_3",
|
||||
"threshold": 1.2368549346675202
|
||||
}
|
||||
```
|
||||
|
||||
## Response body fields
|
||||
|
||||
Field | Description
|
||||
:--- | :---
|
||||
`detector_id` | A unique ID for identifying a detector.
|
||||
`schema_version` | The mapping version of the result index.
|
||||
`data_start_time` | The start of the detection range of the aggregated data.
|
||||
`data_end_time` | The end of the detection range of the aggregated data.
|
||||
`feature_data` | An array of the aggregated data points between the `data_start_time` and `data_end_time`.
|
||||
`execution_start_time` | The actual start time of the detector for a specific run that produces the anomaly result. This start time includes the window delay parameter that you can set to delay data collection. Window delay is the difference between the `execution_start_time` and `data_start_time`.
|
||||
`execution_end_time` | The actual end time of the detector for a specific run that produces the anomaly result.
|
||||
`anomaly_score` | Indicates relative severity of an anomaly. The higher the score, the more anomalous a data point is.
|
||||
`anomaly_grade` | A normalized version of the `anomaly_score` on a scale between 0 and 1.
|
||||
`confidence` | The probability of the accuracy of the `anomaly_score`. The closer this number is to 1, the higher the accuracy. During the probation period of a running detector, the confidence is low (< 0.9) because of its exposure to limited data.
|
||||
`entity` | An entity is a combination of specific category fields’ values. It includes the name and value of the category field. In the previous example, `process_name` is the category field and one of the processes such as `process_3` is the field's value. The `entity` field is only present for a high-cardinality detector (where you've selected a category field).
|
||||
`model_id` | A unique ID that identifies a model. If a detector is a single-stream detector (with no category field), it has only one model. If a detector is a high-cardinality detector (with one or more category fields), it might have multiple models, one for each entity.
|
||||
`threshold` | One of the criteria for a detector to classify a data point as an anomaly is that its `anomaly_score` must surpass a dynamic threshold. This field records the current threshold.
|
||||
|
||||
If an anomaly detector detects an anomaly, the result has the following format:
|
||||
|
||||
```json
|
||||
{
|
||||
"detector_id": "fylE53wBc9MCt6q12tKp",
|
||||
"schema_version": 0,
|
||||
"data_start_time": 1635927900000,
|
||||
"data_end_time": 1635927960000,
|
||||
"feature_data": [
|
||||
{
|
||||
"feature_id": "processing_bytes_max",
|
||||
"feature_name": "processing bytes max",
|
||||
"data": 2291
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_avg",
|
||||
"feature_name": "processing bytes avg",
|
||||
"data": 1677.3333333333333
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_min",
|
||||
"feature_name": "processing bytes min",
|
||||
"data": 1054
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_sum",
|
||||
"feature_name": "processing bytes sum",
|
||||
"data": 5032
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_time_max",
|
||||
"feature_name": "processing time max",
|
||||
"data": 11422
|
||||
}
|
||||
],
|
||||
"anomaly_score": 1.1986675882872033,
|
||||
"anomaly_grade": 0.26806225550178464,
|
||||
"confidence": 0.9607519742565531,
|
||||
"entity": [
|
||||
{
|
||||
"name": "process_name",
|
||||
"value": "process_3"
|
||||
}
|
||||
],
|
||||
"approx_anomaly_start_time": 1635927900000,
|
||||
"relevant_attribution": [
|
||||
{
|
||||
"feature_id": "processing_bytes_max",
|
||||
"data": 0.03628638020431366
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_avg",
|
||||
"data": 0.03384479053991436
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_min",
|
||||
"data": 0.058812549572819096
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_sum",
|
||||
"data": 0.10154576265526988
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_time_max",
|
||||
"data": 0.7695105170276828
|
||||
}
|
||||
],
|
||||
"expected_values": [
|
||||
{
|
||||
"likelihood": 1,
|
||||
"value_list": [
|
||||
{
|
||||
"feature_id": "processing_bytes_max",
|
||||
"data": 2291
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_avg",
|
||||
"data": 1677.3333333333333
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_min",
|
||||
"data": 1054
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_sum",
|
||||
"data": 6062
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_time_max",
|
||||
"data": 23379
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"threshold": 1.0993584705913992,
|
||||
"execution_end_time": 1635898427895,
|
||||
"execution_start_time": 1635898427803
|
||||
}
|
||||
```
|
||||
|
||||
You can see the following additional fields:
|
||||
|
||||
Field | Description
|
||||
:--- | :---
|
||||
`relevant_attribution` | Represents the contribution of each input variable. The sum of the attributions is normalized to 1.
|
||||
`expected_values` | The expected value for each feature.
|
||||
|
||||
At times, the detector might detect an anomaly late.
|
||||
Let's say the detector sees a random mix of the triples {1, 2, 3} and {2, 4, 5} that correspond to `slow weeks` and `busy weeks`, respectively. For example 1, 2, 3, 1, 2, 3, 2, 4, 5, 1, 2, 3, 2, 4, 5, ... and so on.
|
||||
If the detector comes across a pattern {2, 2, X} and it's yet to see X, the detector infers that the pattern is anomalous, but it can't determine at this point which of the 2's is the cause. If X = 3, then the detector knows it's the first 2 in that unfinished triple, and if X = 5, then it's the second 2. If it's the first 2, then the detector detects the anomaly late.
|
||||
|
||||
If a detector detects an anomaly late, the result has the following additional fields:
|
||||
|
||||
Field | Description
|
||||
:--- | :---
|
||||
`past_values` | The actual input that triggered an anomaly. If `past_values` is null, the attributions or expected values are from the current input. If `past_values` is not null, the attributions or expected values are from a past input (for example, the previous two steps of the data [1,2,3]).
|
||||
`approx_anomaly_start_time` | The approximate time of the actual input that triggers an anomaly. This field helps you understand when a detector flags an anomaly. Both single-stream and high-cardinality detectors don't query previous anomaly results because these queries are expensive operations. The cost is especially high for high-cardinality detectors that might have a lot of entities. If the data is not continuous, the accuracy of this field is low and the actual time that the detector detects an anomaly can be earlier.
|
||||
|
||||
```json
|
||||
{
|
||||
"detector_id": "kzcZ43wBgEQAbjDnhzGF",
|
||||
"confidence": 0.9746820962328963,
|
||||
"relevant_attribution": [
|
||||
{
|
||||
"feature_id": "deny_max1",
|
||||
"data": 0.07339452532666227
|
||||
},
|
||||
{
|
||||
"feature_id": "deny_avg",
|
||||
"data": 0.04934972719948845
|
||||
},
|
||||
{
|
||||
"feature_id": "deny_min",
|
||||
"data": 0.01803003656061806
|
||||
},
|
||||
{
|
||||
"feature_id": "deny_sum",
|
||||
"data": 0.14804918212089874
|
||||
},
|
||||
{
|
||||
"feature_id": "accept_max5",
|
||||
"data": 0.7111765287923325
|
||||
}
|
||||
],
|
||||
"task_id": "9Dck43wBgEQAbjDn4zEe",
|
||||
"threshold": 1,
|
||||
"model_id": "kzcZ43wBgEQAbjDnhzGF_entity_app_0",
|
||||
"schema_version": 5,
|
||||
"anomaly_score": 1.141419389056506,
|
||||
"execution_start_time": 1635898427803,
|
||||
"past_values": [
|
||||
{
|
||||
"feature_id": "processing_bytes_max",
|
||||
"data": 905
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_avg",
|
||||
"data": 479
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_min",
|
||||
"data": 128
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_sum",
|
||||
"data": 1437
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_time_max",
|
||||
"data": 8440
|
||||
}
|
||||
],
|
||||
"data_end_time": 1635883920000,
|
||||
"data_start_time": 1635883860000,
|
||||
"feature_data": [
|
||||
{
|
||||
"feature_id": "processing_bytes_max",
|
||||
"feature_name": "processing bytes max",
|
||||
"data": 1360
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_avg",
|
||||
"feature_name": "processing bytes avg",
|
||||
"data": 990
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_min",
|
||||
"feature_name": "processing bytes min",
|
||||
"data": 608
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_sum",
|
||||
"feature_name": "processing bytes sum",
|
||||
"data": 2970
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_time_max",
|
||||
"feature_name": "processing time max",
|
||||
"data": 9670
|
||||
}
|
||||
],
|
||||
"expected_values": [
|
||||
{
|
||||
"likelihood": 1,
|
||||
"value_list": [
|
||||
{
|
||||
"feature_id": "processing_bytes_max",
|
||||
"data": 905
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_avg",
|
||||
"data": 479
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_min",
|
||||
"data": 128
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_bytes_sum",
|
||||
"data": 4847
|
||||
},
|
||||
{
|
||||
"feature_id": "processing_time_max",
|
||||
"data": 15713
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"execution_end_time": 1635898427895,
|
||||
"anomaly_grade": 0.5514172746375128,
|
||||
"entity": [
|
||||
{
|
||||
"name": "process_name",
|
||||
"value": "process_3"
|
||||
}
|
||||
],
|
||||
"approx_anomaly_start_time": 1635883620000
|
||||
}
|
||||
```
|
||||
|
|
@ -24,19 +24,24 @@ PUT _cluster/settings
|
|||
|
||||
Setting | Default | Description
|
||||
:--- | :--- | :---
|
||||
`plugins.anomaly_detection.enabled` | True | Whether the anomaly detection plugin is enabled or not. If disabled, all detectors immediately stop running.
|
||||
`plugins.anomaly_detection.max_anomaly_detectors` | 1,000 | The maximum number of non-high cardinality detectors (no category field) users can create.
|
||||
`plugins.anomaly_detection.max_multi_entity_anomaly_detectors` | 10 | The maximum number of high cardinality detectors (with category field) in a cluster.
|
||||
`plugins.anomaly_detection.max_anomaly_features` | 5 | The maximum number of features for a detector.
|
||||
`plugins.anomaly_detection.ad_result_history_rollover_period` | 12h | How often the rollover condition is checked. If `true`, the plugin rolls over the result index to a new index.
|
||||
`plugins.anomaly_detection.ad_result_history_max_docs` | 250000000 | The maximum number of documents in one result index. The plugin only counts refreshed documents in the primary shards.
|
||||
`plugins.anomaly_detection.ad_result_history_retention_period` | 30d | The maximum age of the result index. If its age exceeds the threshold, the plugin deletes the rolled over result index. If the cluster has only one result index, the plugin keeps the index even if it's older than its configured retention period.
|
||||
`plugins.anomaly_detection.max_entities_per_query` | 1,000 | The maximum unique values per detection interval for high cardinality detectors. By default, if the category field has more than 1,000 unique values in a detector interval, the plugin selects the top 1,000 values and orders them by `doc_count`.
|
||||
`plugins.anomaly_detection.max_entities_for_preview` | 30 | The maximum unique category field values displayed with the preview operation for high cardinality detectors. If the category field has more than 30 unique values, the plugin selects the top 30 values and orders them by `doc_count`.
|
||||
`plugins.anomaly_detection.max_primary_shards` | 10 | The maximum number of primary shards an anomaly detection index can have.
|
||||
`plugins.anomaly_detection.filter_by_backend_roles` | False | When you enable the security plugin and set this to `true`, the plugin filters results based on the user's backend role(s).
|
||||
`plugins.anomaly_detection.max_cache_miss_handling_per_second` | 100 | High cardinality detectors use a cache to store active models. In the event of a cache miss, the cache gets the models from the model checkpoint index. Use this setting to limit the rate of fetching models. Because the thread pool for a GET operation has a queue of 1,000, we recommend setting this value below 1,000.
|
||||
`plugins.anomaly_detection.max_batch_task_per_node` | 2 | Starting a historical detector triggers a batch task. This setting is the number of batch tasks that you can run per data node. You can tune this setting from 1 to 1000. If the data nodes can't support all batch tasks and you're not sure if the data nodes are capable of running more historical detectors, add more data nodes instead of changing this setting to a higher value.
|
||||
`plugins.anomaly_detection.max_old_ad_task_docs_per_detector` | 10 | You can run the same historical detector many times. For each run, the anomaly detection plugin creates a new task. This setting is the number of previous tasks the plugin keeps. Set this value to at least 1 to track its last run. You can keep a maximum of 1,000 old tasks to avoid overwhelming the cluster.
|
||||
`plugins.anomaly_detection.batch_task_piece_size` | 1000 | The date range for a historical task is split into smaller pieces and the anomaly detection plugin runs the task piece by piece. Each piece contains 1,000 detection intervals by default. For example, if detector interval is 1 minute and one piece is 1000 minutes, the feature data is queried every 1,000 minutes. You can change this setting from 1 to 10,000.
|
||||
`plugins.anomaly_detection.batch_task_piece_interval_seconds` | 5 | Add a time interval between historical detector tasks. This interval prevents the task from consuming too much of the available resources and starving other operations like search and bulk index. You can change this setting from 1 to 600 seconds.
|
||||
plugins.anomaly_detection.enabled | True | Whether the anomaly detection plugin is enabled or not. If disabled, all detectors immediately stop running.
|
||||
plugins.anomaly_detection.max_anomaly_detectors | 1,000 | The maximum number of non-high cardinality detectors (no category field) users can create.
|
||||
plugins.anomaly_detection.max_multi_entity_anomaly_detectors | 10 | The maximum number of high cardinality detectors (with category field) in a cluster.
|
||||
plugins.anomaly_detection.max_anomaly_features | 5 | The maximum number of features for a detector.
|
||||
plugins.anomaly_detection.ad_result_history_rollover_period | 12h | How often the rollover condition is checked. If `true`, the anomaly detection plugin rolls over the result index to a new index.
|
||||
plugins.anomaly_detection.ad_result_history_max_docs_per_shard | 1,350,000,000 | The maximum number of documents in a single shard of the result index. The anomaly detection plugin only counts the refreshed documents in the primary shards.
|
||||
plugins.anomaly_detection.max_entities_per_query | 1,000,000 | The maximum unique values per detection interval for high cardinality detectors. By default, if the category field(s) have more than the configured unique values in a detector interval, the anomaly detection plugin orders them by the natural ordering of categorical values (for example, entity `ab` comes before `bc`) and then selects the top values.
|
||||
plugins.anomaly_detection.max_entities_for_preview | 5 | The maximum unique category field values displayed with the preview operation for high cardinality detectors. By default, if the category field(s) have more than the configured unique values in a detector interval, the anomaly detection plugin orders them by the natural ordering of categorical values (for example, entity `ab` comes before `bc`) and then selects the top values.
|
||||
plugins.anomaly_detection.max_primary_shards | 10 | The maximum number of primary shards an anomaly detection index can have.
|
||||
plugins.anomaly_detection.filter_by_backend_roles | False | When you enable the security plugin and set this to `true`, the anomaly detection plugin filters results based on the user's backend role(s).
|
||||
plugins.anomaly_detection.max_batch_task_per_node | 10 | Starting a historical analysis triggers a batch task. This setting is the number of batch tasks that you can run per data node. You can tune this setting from 1 to 1,000. If the data nodes can’t support all batch tasks and you’re not sure if the data nodes are capable of running more historical analysis, add more data nodes instead of changing this setting to a higher value. Increasing this value might bring more load on each data node.
|
||||
plugins.anomaly_detection.max_old_ad_task_docs_per_detector | 1 | You can run historical analysis for the same detector many times. For each run, the anomaly detection plugin creates a new task. This setting is the number of previous tasks the plugin keeps. Set this value to at least 1 to track its last run. You can keep a maximum of 1,000 old tasks to avoid overwhelming the cluster.
|
||||
plugins.anomaly_detection.batch_task_piece_size | 1,000 | The date range for a historical task is split into smaller pieces and the anomaly detection plugin runs the task piece by piece. Each piece contains 1,000 detection intervals by default. For example, if detector interval is 1 minute and one piece is 1,000 minutes, the feature data is queried every 1,000 minutes. You can change this setting from 1 to 10,000.
|
||||
plugins.anomaly_detection.batch_task_piece_interval_seconds | 5 | Add a time interval between two pieces of the same historical analysis task. This interval prevents the task from consuming too much of the available resources and starving other operations like search and bulk index. You can change this setting from 1 to 600 seconds.
|
||||
plugins.anomaly_detection.max_top_entities_for_historical_analysis | 1,000 | The maximum number of top entities that you run for a high cardinality detector historical analysis. The range is from 1 to 10,000.
|
||||
plugins.anomaly_detection.max_running_entities_per_detector_for_historical_analysis | 10 | The number of entity tasks that you can run in parallel for a high cardinality detector analysis. The task slots available on your cluster also impact how many entities run in parallel. If a cluster has 3 data nodes, each data node has 10 task slots by default. Say you already have two high cardinality detectors and each of them run 10 entities. If you start a single-entity detector that takes 1 task slot, the number of task slots available is 10 * 3 - 10 * 2 - 1 = 9. If you now start a new high cardinality detector, the detector can only run 9 entities in parallel and not 10. You can tune this value from 1 to 1,000 based on your cluster's capability. If you set a higher value, the anomaly detection plugin runs historical analysis faster but also consumes more resources.
|
||||
plugins.anomaly_detection.max_cached_deleted_tasks | 1,000 | You can rerun historical analysis for a single detector as many times as you like. The anomaly detection plugin only keeps a limited number of old tasks, by default 1 old task. If you run historical analysis three times for a detector, the oldest task is deleted. Because historical analysis generates a number of anomaly results in a short span of time, it's necessary to clean up anomaly results for a deleted task. With this field, you can configure how many deleted tasks you can cache at most. The plugin cleans up a task's results when it's deleted. If the plugin fails to do this cleanup, it adds the task's results into a cache and an hourly cron job performs the cleanup. You can use this setting to limit how many old tasks are put into cache to avoid a DDoS attack. After an hour, if still you find an old task result in the cache, use the [delete detector results API]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/api/#delete-detector-results) to delete the task result manually. You can tune this setting from 1 to 10,000.
|
||||
plugins.anomaly_detection.delete_anomaly_result_when_delete_detector | False | Whether the anomaly detection plugin deletes the anomaly result when you delete a detector. If you want to save some disk space, especially if you've high cardinality detectors generating a lot of results, set this field to true. Alternatively, you can use the [delete detector results API]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/api/#delete-detector-results) to manually delete the results.
|
||||
plugins.anomaly_detection.dedicated_cache_size | 10 | If the real-time analysis of a high cardinality detector starts successfully, the anomaly detection plugin guarantees keeping 10 (dynamically adjustable via this setting) entities' models in memory per node. If the number of entities exceeds this limit, the plugin puts the extra entities' models in a memory space shared by all detectors. The actual number of entities varies based on the memory that you've available and the frequencies of the entities. If you'd like the plugin to guarantee keeping more entities' models in memory and if you're cluster has sufficient memory, you can increase this setting value.
|
||||
plugins.anomaly_detection.max_concurrent_preview | 2 | The maximum number of concurrent previews. You can use this setting to limit resource usage.
|
||||
plugins.anomaly_detection.model_max_size_percent | 0.1 | The upper bound of the memory percentage for a model.
|
||||
|
|
|
|||
|
|
@ -19,10 +19,12 @@ Use the alerting API to programmatically manage monitors and alerts.
|
|||
|
||||
---
|
||||
|
||||
## Create monitor
|
||||
## Create query-level monitor
|
||||
Introduced 1.0
|
||||
{: .label .label-purple }
|
||||
|
||||
Query-level monitors run the query and check whether the results should trigger any alerts. As such, query-level monitors can only trigger one alert at a time. For more information about query-level monitors versus bucket-level monitors, see [Create monitors]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/monitors/#create-monitors).
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
|
|
@ -30,6 +32,7 @@ POST _plugins/_alerting/monitors
|
|||
{
|
||||
"type": "monitor",
|
||||
"name": "test-monitor",
|
||||
"monitor_type": "query_level_monitor",
|
||||
"enabled": true,
|
||||
"schedule": {
|
||||
"period": {
|
||||
|
|
@ -166,7 +169,7 @@ If you use a custom webhook for your destination and need to embed JSON in the m
|
|||
},
|
||||
"throttle_enabled": false,
|
||||
"subject_template": {
|
||||
"source": "TheSubject",
|
||||
"source": "Subject",
|
||||
"lang": "mustache"
|
||||
}
|
||||
}]
|
||||
|
|
@ -186,6 +189,7 @@ The following example creates a monitor that runs at 12:10 PM Pacific Time on th
|
|||
{
|
||||
"type": "monitor",
|
||||
"name": "test-monitor",
|
||||
"monitor_type": "query_level_monitor",
|
||||
"enabled": true,
|
||||
"schedule": {
|
||||
"cron" : {
|
||||
|
|
@ -228,7 +232,7 @@ The following example creates a monitor that runs at 12:10 PM Pacific Time on th
|
|||
"name": "test-action",
|
||||
"destination_id": "ld7912sBlQ5JUWWFThoW",
|
||||
"message_template": {
|
||||
"source": "This is my message body."
|
||||
"source": "This is a message body."
|
||||
},
|
||||
"throttle_enabled": true,
|
||||
"throttle": {
|
||||
|
|
@ -236,7 +240,7 @@ The following example creates a monitor that runs at 12:10 PM Pacific Time on th
|
|||
"unit": "MINUTES"
|
||||
},
|
||||
"subject_template": {
|
||||
"source": "TheSubject"
|
||||
"source": "Subject"
|
||||
}
|
||||
}]
|
||||
}]
|
||||
|
|
@ -247,11 +251,270 @@ For a full list of timezone names, refer to [Wikipedia](https://en.wikipedia.org
|
|||
|
||||
---
|
||||
|
||||
## Create bucket-level monitor
|
||||
|
||||
Bucket-level monitors categorize results into buckets separated by fields. The monitor then runs your script with each bucket's results and evaluates whether to trigger an alert. For more information about bucket-level monitors versus query-level monitors, see [Create monitors]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/monitors/#create-monitors).
|
||||
|
||||
```json
|
||||
POST _plugins/_alerting/monitors
|
||||
{
|
||||
"type": "monitor",
|
||||
"name": "test-bucket-level-monitor",
|
||||
"monitor_type": "bucket_level_monitor",
|
||||
"enabled": true,
|
||||
"schedule": {
|
||||
"period": {
|
||||
"interval": 1,
|
||||
"unit": "MINUTES"
|
||||
}
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"search": {
|
||||
"indices": [
|
||||
"movies"
|
||||
],
|
||||
"query": {
|
||||
"size": 0,
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{
|
||||
"range": {
|
||||
"order_date": {
|
||||
"from": "{{period_end}}||-1h",
|
||||
"to": "{{period_end}}",
|
||||
"include_lower": true,
|
||||
"include_upper": true,
|
||||
"format": "epoch_millis"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"aggregations": {
|
||||
"composite_agg": {
|
||||
"composite": {
|
||||
"sources": [
|
||||
{
|
||||
"user": {
|
||||
"terms": {
|
||||
"field": "user"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"aggregations": {
|
||||
"avg_products_base_price": {
|
||||
"avg": {
|
||||
"field": "products.base_price"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"triggers": [
|
||||
{
|
||||
"bucket_level_trigger": {
|
||||
"name": "test-trigger",
|
||||
"severity": "1",
|
||||
"condition": {
|
||||
"buckets_path": {
|
||||
"_count": "_count",
|
||||
"avg_products_base_price": "avg_products_base_price"
|
||||
},
|
||||
"parent_bucket_path": "composite_agg",
|
||||
"script": {
|
||||
"source": "params._count > 50 || params.avg_products_base_price < 35",
|
||||
"lang": "painless"
|
||||
}
|
||||
},
|
||||
"actions": [
|
||||
{
|
||||
"name": "test-action",
|
||||
"destination_id": "E4o5hnsB6KjPKmHtpfCA",
|
||||
"message_template": {
|
||||
"source": """Monitor {{ctx.monitor.name}} just entered alert status. Please investigate the issue. - Trigger: {{ctx.trigger.name}} - Severity: {{ctx.trigger.severity}} - Period start: {{ctx.periodStart}} - Period end: {{ctx.periodEnd}} - Deduped Alerts: {{ctx.dedupedAlerts}} * {{id}} : {{bucket_keys}} {{ctx.dedupedAlerts}} - New Alerts: {{ctx.newAlerts}} * {{id}} : {{bucket_keys}} {{ctx.newAlerts}} - Completed Alerts: {{ctx.completedAlerts}} * {{id}} : {{bucket_keys}} {{ctx.completedAlerts}}""",
|
||||
"lang": "mustache"
|
||||
},
|
||||
"throttle_enabled": false,
|
||||
"throttle": {
|
||||
"value": 10,
|
||||
"unit": "MINUTES"
|
||||
},
|
||||
"action_execution_policy": {
|
||||
"action_execution_scope": {
|
||||
"per_alert": {
|
||||
"actionable_alerts": [
|
||||
"DEDUPED",
|
||||
"NEW"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"subject_template": {
|
||||
"source": "The Subject",
|
||||
"lang": "mustache"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### Sample response
|
||||
```json
|
||||
{
|
||||
"_id" : "Dfxr63sBwex6DxEhHV5N",
|
||||
"_version" : 1,
|
||||
"_seq_no" : 3,
|
||||
"_primary_term" : 1,
|
||||
"monitor" : {
|
||||
"type" : "monitor",
|
||||
"schema_version" : 4,
|
||||
"name" : "test-bucket-level-monitor",
|
||||
"monitor_type" : "bucket_level_monitor",
|
||||
"user" : {
|
||||
"name" : "",
|
||||
"backend_roles" : [ ],
|
||||
"roles" : [ ],
|
||||
"custom_attribute_names" : [ ],
|
||||
"user_requested_tenant" : null
|
||||
},
|
||||
"enabled" : true,
|
||||
"enabled_time" : 1631742270785,
|
||||
"schedule" : {
|
||||
"period" : {
|
||||
"interval" : 1,
|
||||
"unit" : "MINUTES"
|
||||
}
|
||||
},
|
||||
"inputs" : [
|
||||
{
|
||||
"search" : {
|
||||
"indices" : [
|
||||
"opensearch_dashboards_sample_data_flights"
|
||||
],
|
||||
"query" : {
|
||||
"size" : 0,
|
||||
"query" : {
|
||||
"bool" : {
|
||||
"filter" : [
|
||||
{
|
||||
"range" : {
|
||||
"order_date" : {
|
||||
"from" : "{{period_end}}||-1h",
|
||||
"to" : "{{period_end}}",
|
||||
"include_lower" : true,
|
||||
"include_upper" : true,
|
||||
"format" : "epoch_millis",
|
||||
"boost" : 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"adjust_pure_negative" : true,
|
||||
"boost" : 1.0
|
||||
}
|
||||
},
|
||||
"aggregations" : {
|
||||
"composite_agg" : {
|
||||
"composite" : {
|
||||
"size" : 10,
|
||||
"sources" : [
|
||||
{
|
||||
"user" : {
|
||||
"terms" : {
|
||||
"field" : "user",
|
||||
"missing_bucket" : false,
|
||||
"order" : "asc"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"aggregations" : {
|
||||
"avg_products_base_price" : {
|
||||
"avg" : {
|
||||
"field" : "products.base_price"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"triggers" : [
|
||||
{
|
||||
"bucket_level_trigger" : {
|
||||
"id" : "C_xr63sBwex6DxEhHV5B",
|
||||
"name" : "test-trigger",
|
||||
"severity" : "1",
|
||||
"condition" : {
|
||||
"buckets_path" : {
|
||||
"_count" : "_count",
|
||||
"avg_products_base_price" : "avg_products_base_price"
|
||||
},
|
||||
"parent_bucket_path" : "composite_agg",
|
||||
"script" : {
|
||||
"source" : "params._count > 50 || params.avg_products_base_price < 35",
|
||||
"lang" : "painless"
|
||||
},
|
||||
"gap_policy" : "skip"
|
||||
},
|
||||
"actions" : [
|
||||
{
|
||||
"id" : "DPxr63sBwex6DxEhHV5B",
|
||||
"name" : "test-action",
|
||||
"destination_id" : "E4o5hnsB6KjPKmHtpfCA",
|
||||
"message_template" : {
|
||||
"source" : "Monitor {{ctx.monitor.name}} just entered alert status. Please investigate the issue. - Trigger: {{ctx.trigger.name}} - Severity: {{ctx.trigger.severity}} - Period start: {{ctx.periodStart}} - Period end: {{ctx.periodEnd}} - Deduped Alerts: {{ctx.dedupedAlerts}} * {{id}} : {{bucket_keys}} {{ctx.dedupedAlerts}} - New Alerts: {{ctx.newAlerts}} * {{id}} : {{bucket_keys}} {{ctx.newAlerts}} - Completed Alerts: {{ctx.completedAlerts}} * {{id}} : {{bucket_keys}} {{ctx.completedAlerts}}",
|
||||
"lang" : "mustache"
|
||||
},
|
||||
"throttle_enabled" : false,
|
||||
"subject_template" : {
|
||||
"source" : "The Subject",
|
||||
"lang" : "mustache"
|
||||
},
|
||||
"throttle" : {
|
||||
"value" : 10,
|
||||
"unit" : "MINUTES"
|
||||
},
|
||||
"action_execution_policy" : {
|
||||
"action_execution_scope" : {
|
||||
"per_alert" : {
|
||||
"actionable_alerts" : [
|
||||
"DEDUPED",
|
||||
"NEW"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"last_update_time" : 1631742270785
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Update monitor
|
||||
Introduced 1.0
|
||||
{: .label .label-purple }
|
||||
|
||||
When you update a monitor, include the current version number as a parameter. OpenSearch increments the version number automatically (see the sample response).
|
||||
When updating a monitor, you can optionally include `seq_no` and `primary_term` as URL parameters. If these numbers don't match the existing monitor or the monitor doesn't exist, the alerting plugin throws an error. OpenSearch increments the version number and the sequence number automatically (see the sample response).
|
||||
|
||||
#### Request
|
||||
|
||||
|
|
@ -307,6 +570,58 @@ PUT _plugins/_alerting/monitors/<monitor_id>
|
|||
}],
|
||||
"last_update_time": 1551466639295
|
||||
}
|
||||
|
||||
PUT _plugins/_alerting/monitors/<monitor_id>?if_seq_no=3&if_primary_term=1
|
||||
{
|
||||
"type": "monitor",
|
||||
"name": "test-monitor",
|
||||
"enabled": true,
|
||||
"enabled_time": 1551466220455,
|
||||
"schedule": {
|
||||
"period": {
|
||||
"interval": 1,
|
||||
"unit": "MINUTES"
|
||||
}
|
||||
},
|
||||
"inputs": [{
|
||||
"search": {
|
||||
"indices": [
|
||||
"*"
|
||||
],
|
||||
"query": {
|
||||
"query": {
|
||||
"match_all": {
|
||||
"boost": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}],
|
||||
"triggers": [{
|
||||
"id": "StaeOmkBC25HCRGmL_y-",
|
||||
"name": "test-trigger",
|
||||
"severity": "1",
|
||||
"condition": {
|
||||
"script": {
|
||||
"source": "return true",
|
||||
"lang": "painless"
|
||||
}
|
||||
},
|
||||
"actions": [{
|
||||
"name": "test-action",
|
||||
"destination_id": "RtaaOmkBC25HCRGm0fxi",
|
||||
"subject_template": {
|
||||
"source": "My Message Subject",
|
||||
"lang": "mustache"
|
||||
},
|
||||
"message_template": {
|
||||
"source": "This is my message body.",
|
||||
"lang": "mustache"
|
||||
}
|
||||
}]
|
||||
}],
|
||||
"last_update_time": 1551466639295
|
||||
}
|
||||
```
|
||||
|
||||
#### Sample response
|
||||
|
|
@ -315,6 +630,8 @@ PUT _plugins/_alerting/monitors/<monitor_id>
|
|||
{
|
||||
"_id": "Q9aXOmkBC25HCRGmzfw-",
|
||||
"_version": 4,
|
||||
"_seq_no": 4,
|
||||
"_primary_term": 1,
|
||||
"monitor": {
|
||||
"type": "monitor",
|
||||
"name": "test-monitor",
|
||||
|
|
@ -387,6 +704,8 @@ GET _plugins/_alerting/monitors/<monitor_id>
|
|||
{
|
||||
"_id": "Q9aXOmkBC25HCRGmzfw-",
|
||||
"_version": 3,
|
||||
"_seq_no": 3,
|
||||
"_primary_term": 1,
|
||||
"monitor": {
|
||||
"type": "monitor",
|
||||
"name": "test-monitor",
|
||||
|
|
@ -960,7 +1279,9 @@ POST _plugins/_alerting/destinations
|
|||
```json
|
||||
{
|
||||
"_id": "nO-yFmkB8NzS6aXjJdiI",
|
||||
"_version": 1,
|
||||
"_version" : 1,
|
||||
"_seq_no" : 3,
|
||||
"_primary_term" : 1,
|
||||
"destination": {
|
||||
"type": "slack",
|
||||
"name": "my-destination",
|
||||
|
|
@ -979,6 +1300,8 @@ POST _plugins/_alerting/destinations
|
|||
Introduced 1.0
|
||||
{: .label .label-purple }
|
||||
|
||||
When updating a destination, you can optionally include `seq_no` and `primary_term` as URL parameters. If these numbers don't match the existing destination or the destination doesn't exist, the alerting plugin throws an error. OpenSearch increments the version number and the sequence number automatically (see the sample response).
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
|
|
@ -990,6 +1313,15 @@ PUT _plugins/_alerting/destinations/<destination-id>
|
|||
"url": "http://www.example.com"
|
||||
}
|
||||
}
|
||||
|
||||
PUT _plugins/_alerting/destinations/<destination-id>?if_seq_no=3&if_primary_term=1
|
||||
{
|
||||
"name": "my-updated-destination",
|
||||
"type": "slack",
|
||||
"slack": {
|
||||
"url": "http://www.example.com"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Sample response
|
||||
|
|
@ -997,7 +1329,9 @@ PUT _plugins/_alerting/destinations/<destination-id>
|
|||
```json
|
||||
{
|
||||
"_id": "pe-1FmkB8NzS6aXjqvVY",
|
||||
"_version": 4,
|
||||
"_version" : 2,
|
||||
"_seq_no" : 4,
|
||||
"_primary_term" : 1,
|
||||
"destination": {
|
||||
"type": "slack",
|
||||
"name": "my-updated-destination",
|
||||
|
|
@ -1175,6 +1509,8 @@ POST _plugins/_alerting/destinations/email_accounts
|
|||
Introduced 1.0
|
||||
{: .label .label-purple }
|
||||
|
||||
When updating an email account, you can optionally include `seq_no` and `primary_term` as URL parameters. If these numbers don't match the existing email account or the email account doesn't exist, the alerting plugin throws an error. OpenSearch increments the version number and the sequence number automatically (see the sample response).
|
||||
|
||||
#### Request
|
||||
```json
|
||||
PUT _plugins/_alerting/destinations/email_accounts/<email_account_id>
|
||||
|
|
@ -1185,6 +1521,15 @@ PUT _plugins/_alerting/destinations/email_accounts/<email_account_id>
|
|||
"port": 465,
|
||||
"method": "ssl"
|
||||
}
|
||||
|
||||
PUT _plugins/_alerting/destinations/email_accounts/<email_account_id>?if_seq_no=18&if_primary_term=2
|
||||
{
|
||||
"name": "example_account",
|
||||
"email": "example@email.com",
|
||||
"host": "smtp.email.com",
|
||||
"port": 465,
|
||||
"method": "ssl"
|
||||
}
|
||||
```
|
||||
#### Sample response
|
||||
```json
|
||||
|
|
@ -1373,6 +1718,8 @@ POST _plugins/_alerting/destinations/email_groups
|
|||
Introduced 1.0
|
||||
{: .label .label-purple }
|
||||
|
||||
When updating an email group, you can optionally include `seq_no` and `primary_term` as URL parameters. If these numbers don't match the existing email group or the email group doesn't exist, the alerting plugin throws an error. OpenSearch increments the version number and the sequence number automatically (see the sample response).
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
|
|
@ -1383,6 +1730,14 @@ PUT _plugins/_alerting/destinations/email_groups/<email_group_id>
|
|||
"email": "example@email.com"
|
||||
}]
|
||||
}
|
||||
|
||||
PUT _plugins/_alerting/destinations/email_groups/<email_group_id>?if_seq_no=16&if_primary_term=2
|
||||
{
|
||||
"name": "example_email_group",
|
||||
"emails": [{
|
||||
"email": "example@email.com"
|
||||
}]
|
||||
}
|
||||
```
|
||||
#### Sample response
|
||||
|
||||
|
|
|
|||
|
|
@ -19,11 +19,11 @@ has_children: false
|
|||
|
||||
Term | Definition
|
||||
:--- | :---
|
||||
Monitor | A job that runs on a defined schedule and queries OpenSearch. The results of these queries are then used as input for one or more *triggers*.
|
||||
Monitor | A job that runs on a defined schedule and queries OpenSearch indices. The results of these queries are then used as input for one or more *triggers*.
|
||||
Trigger | Conditions that, if met, generate *alerts*.
|
||||
Alert | An event associated with a trigger. When an alert is created, the trigger performs *actions*, which can include sending a notification.
|
||||
Action | The information that you want the monitor to send out after being triggered. Actions have a *destination*, a message subject, and a message body.
|
||||
Destination | A reusable location for an action, such as Amazon Chime, Slack, or a webhook URL.
|
||||
Destination | A reusable location for an action. Supported locations are Amazon Chime, Email, Slack, or custom webhook.
|
||||
|
||||
|
||||
---
|
||||
|
|
@ -34,9 +34,9 @@ Destination | A reusable location for an action, such as Amazon Chime, Slack, or
|
|||
1. Specify a name for the destination so that you can identify it later.
|
||||
1. For **Type**, choose Slack, Amazon Chime, custom webhook, or [email](#email-as-a-destination).
|
||||
|
||||
For Email type, refer to [Email as a destination](#email-as-a-destination) section below. For all other types, specify the webhook URL. For more information about webhooks, see the documentation for [Slack](https://api.slack.com/incoming-webhooks) and [Amazon Chime](https://docs.aws.amazon.com/chime/latest/ug/webhooks.html).
|
||||
For Email, refer to the [Email as a destination](#email-as-a-destination) section below. For all other types, specify the webhook URL. See the documentation for [Slack](https://api.slack.com/incoming-webhooks) and [Amazon Chime](https://docs.aws.amazon.com/chime/latest/ug/webhooks.html) to learn more about webhooks.
|
||||
|
||||
For custom webhooks, you must specify more information: parameters and headers. For example, if your endpoint requires basic authentication, you might need to add a header with a key of `Authorization` and a value of `Basic <Base64-encoded-credential-string>`. You might also need to change `Content-Type` to whatever your webhook requires. Popular values are `application/json`, `application/xml`, and `text/plain`.
|
||||
If you're using custom webhooks, you must specify more information: parameters and headers. For example, if your endpoint requires basic authentication, you might need to add a header with a key of `Authorization` and a value of `Basic <Base64-encoded-credential-string>`. You might also need to change `Content-Type` to whatever your webhook requires. Popular values are `application/json`, `application/xml`, and `text/plain`.
|
||||
|
||||
This information is stored in plain text in the OpenSearch cluster. We will improve this design in the future, but for now, the encoded credentials (which are neither encrypted nor hashed) might be visible to other OpenSearch users.
|
||||
|
||||
|
|
@ -55,7 +55,7 @@ To configure a sender email, do the following:
|
|||
1. After you choose **Email** as the destination type, choose **Manage senders**.
|
||||
1. Choose **Add sender**, **New sender** and enter a unique name.
|
||||
1. Enter the email address, SMTP host (e.g. `smtp.gmail.com` for a Gmail account), and the port.
|
||||
1. Choose an encryption method, or use the default value of **None**. However, most email providers require SSL or TLS, which requires a username and password in OpenSearch keystore. Refer to [Authenticate sender account](#authenticate-sender-account) to learn more.
|
||||
1. Choose an encryption method, or use the default value of **None**. However, most email providers require SSL or TLS, which require a username and password in OpenSearch keystore. Refer to [Authenticate sender account](#authenticate-sender-account) to learn more.
|
||||
1. Choose **Save** to save the configuration and create the sender. You can create a sender even before you add your credentials to the OpenSearch keystore. However, you must [authenticate each sender account](#authenticate-sender-account) before you use the destination to send your alert.
|
||||
|
||||
You can reuse senders across many different destinations, but each destination only supports one sender.
|
||||
|
|
@ -82,7 +82,7 @@ If your email provider requires SSL or TLS, you must authenticate each sender ac
|
|||
./bin/opensearch-keystore add plugins.alerting.destination.email.<sender_name>.password
|
||||
```
|
||||
|
||||
**Note**: Keystore settings are node-specific. You must run these commands on each node.
|
||||
Note: Keystore settings are node-specific. You must run these commands on each node.
|
||||
{: .note}
|
||||
|
||||
To change or update your credentials (after you've added them to the keystore on every node), call the reload API to automatically update those credentials without restarting OpenSearch:
|
||||
|
|
@ -101,20 +101,9 @@ POST _nodes/reload_secure_settings
|
|||
|
||||
1. Choose **Alerting**, **Monitors**, **Create monitor**.
|
||||
1. Specify a name for the monitor.
|
||||
1. Choose either **Per query monitor** or **Per bucket monitor**.
|
||||
|
||||
The anomaly detection option is for pairing with the anomaly detection plugin. See [Anomaly Detection]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/).
|
||||
For anomaly detector, choose an appropriate schedule for the monitor based on the detector interval. Otherwise, the alerting monitor might miss reading the results.
|
||||
|
||||
For example, assume you set the monitor interval and the detector interval as 5 minutes, and you start the detector at 12:00. If an anomaly is detected at 12:05, it might be available at 12:06 because of the delay between writing the anomaly and it being available for queries. The monitor reads the anomaly results between 12:00 and 12:05, so it does not get the anomaly results available at 12:06.
|
||||
|
||||
To avoid this issue, make sure the alerting monitor is at least twice the detector interval.
|
||||
When you create a monitor using OpenSearch Dashboards, the anomaly detector plugin generates a default monitor schedule that's twice the detector interval.
|
||||
|
||||
Whenever you update a detector’s interval, make sure to update the associated monitor interval as well, as the anomaly detection plugin does not do this automatically.
|
||||
|
||||
1. Choose one or more indices. You can also use `*` as a wildcard to specify an index pattern.
|
||||
|
||||
If you use the security plugin, you can only choose indices that you have permission to access. For details, see [Alerting security]({{site.url}}{{site.baseurl}}/security-plugin/).
|
||||
Whereas query-level monitors run your specified query and then check whether the query's results triggers any alerts, bucket-level monitors let you select fields to create buckets and categorize your results into those buckets. The alerting plugin runs each bucket's unique results against a script you define later, so you have finer control over which results should trigger alerts. Each of those buckets can trigger an alert, but query-level monitors can only trigger one alert at a time.
|
||||
|
||||
1. Define the monitor in one of three ways: visually, using a query, or using an anomaly detector.
|
||||
|
||||
|
|
@ -167,39 +156,57 @@ Whenever you update a detector’s interval, make sure to update the associated
|
|||
}
|
||||
```
|
||||
|
||||
"Start" and "end" refer to the interval at which the monitor runs. See [Available variables](#available-variables).
|
||||
"Start" and "end" refer to the interval at which the monitor runs. See [Available variables](#available-variables).
|
||||
|
||||
To define a monitor visually, choose **Visual editor**. Then choose a source index, a timeframe, an aggregation (for example, `count()` or `average()`), a data filter if you want to monitor a subset of your source index, and a group-by field if you want to include an aggregation field in your query. At least one group-by field is required if you're defining a bucket-level monitor. Visual definition works well for most monitors.
|
||||
|
||||
1. To define a monitor visually, choose **Define using visual graph**. Then choose an aggregation (for example, `count()` or `average()`), a set of documents, and a timeframe. Visual definition works well for most monitors.
|
||||
If you use the security plugin, you can only choose indices that you have permission to access. For details, see [Alerting security]({{site.url}}{{site.baseurl}}/security-plugin/).
|
||||
|
||||
To use a query, choose **Define using extraction query**, add your query (using [the OpenSearch query DSL]({{site.url}}{{site.baseurl}}/opensearch/query-dsl/full-text/)), and test it using the **Run** button.
|
||||
To use a query, choose **Extraction query editor**, add your query (using [the OpenSearch query DSL]({{site.url}}{{site.baseurl}}/opensearch/query-dsl/full-text/)), and test it using the **Run** button.
|
||||
|
||||
The monitor makes this query to OpenSearch as often as the schedule dictates; check the **Query Performance** section and make sure you're comfortable with the performance implications.
|
||||
The monitor makes this query to OpenSearch as often as the schedule dictates; check the **Query Performance** section and make sure you're comfortable with the performance implications.
|
||||
|
||||
To use an anomaly detector, choose **Anomaly detector** and select your **Detector**.
|
||||
|
||||
The anomaly detection option is for pairing with the anomaly detection plugin. See [Anomaly Detection]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/).
|
||||
For anomaly detector, choose an appropriate schedule for the monitor based on the detector interval. Otherwise, the alerting monitor might miss reading the results.
|
||||
|
||||
For example, assume you set the monitor interval and the detector interval as 5 minutes, and you start the detector at 12:00. If an anomaly is detected at 12:05, it might be available at 12:06 because of the delay between writing the anomaly and it being available for queries. The monitor reads the anomaly results between 12:00 and 12:05, so it does not get the anomaly results available at 12:06.
|
||||
|
||||
To avoid this issue, make sure the alerting monitor is at least twice the detector interval.
|
||||
When you create a monitor using OpenSearch Dashboards, the anomaly detector plugin generates a default monitor schedule that's twice the detector interval.
|
||||
|
||||
Whenever you update a detector’s interval, make sure to update the associated monitor interval as well, as the anomaly detection plugin does not do this automatically.
|
||||
|
||||
**Note**: Anomaly detection is available only if you are defining a per query monitor.
|
||||
{: .note}
|
||||
|
||||
To use an anomaly detector, choose **Define using Anomaly detector** and select your **Detector**.
|
||||
1. Choose a frequency and timezone for your monitor. Note that you can only pick a timezone if you choose Daily, Weekly, Monthly, or [custom cron expression]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/cron/) for frequency.
|
||||
1. Choose **Create**.
|
||||
|
||||
1. Add a trigger to your monitor.
|
||||
|
||||
---
|
||||
|
||||
## Create triggers
|
||||
|
||||
The next step in creating a monitor is to create a trigger. These steps differ depending on whether you chose **Define using visual graph** or **Define using extraction query** or **Define using Anomaly detector** when you created the monitor.
|
||||
Steps to create a trigger differ depending on whether you chose **Visual editor**, **Extraction query editor**, or **Anomaly detector** when you created the monitor.
|
||||
|
||||
Either way, you begin by specifying a name and severity level for the trigger. Severity levels help you manage alerts. A trigger with a high severity level (e.g. 1) might page a specific individual, whereas a trigger with a low severity level might message a chat room.
|
||||
You begin by specifying a name and severity level for the trigger. Severity levels help you manage alerts. A trigger with a high severity level (e.g. 1) might page a specific individual, whereas a trigger with a low severity level might message a chat room.
|
||||
|
||||
Remember that query-level monitors run your trigger's script just once against the query's results, but bucket-level monitors execute your trigger's script on each bucket, so you should create a trigger that best fits the monitor you chose. If you want to execute multiple scripts, you must create multiple triggers.
|
||||
|
||||
### Visual graph
|
||||
### Visual editor
|
||||
|
||||
For **Trigger condition**, specify a threshold for the aggregation and timeframe you chose earlier, such as "is below 1,000" or "is exactly 10."
|
||||
For a query-level monitor's **Trigger condition**, specify a threshold for the aggregation and timeframe you chose earlier, such as "is below 1,000" or "is exactly 10."
|
||||
|
||||
The line moves up and down as you increase and decrease the threshold. Once this line is crossed, the trigger evaluates to true.
|
||||
|
||||
Bucket-level monitors also require you to specify a threshold and value for your aggregation and timeframe, but you can use a maximum of five conditions to better refine your trigger. Optionally, you can also use a keyword filter to filter for a specific field in your index.
|
||||
|
||||
|
||||
### Extraction query
|
||||
|
||||
For **Trigger condition**, specify a Painless script that returns true or false. Painless is the default OpenSearch scripting language and has a syntax similar to Groovy.
|
||||
If you're using a query-level monitor, specify a Painless script that returns true or false. Painless is the default OpenSearch scripting language and has a syntax similar to Groovy.
|
||||
|
||||
Trigger condition scripts revolve around the `ctx.results[0]` variable, which corresponds to the extraction query response. For example, your script might reference `ctx.results[0].hits.total.value` or `ctx.results[0].hits.hits[i]._source.error_code`.
|
||||
|
||||
|
|
@ -208,6 +215,27 @@ A return value of true means the trigger condition has been met, and the trigger
|
|||
The **Info** link next to **Trigger condition** contains a useful summary of the variables and results available to your query.
|
||||
{: .tip }
|
||||
|
||||
Bucket-level monitors require you to specify more information in your trigger condition. At a minimum, you must have the following fields:
|
||||
|
||||
- `buckets_path`, which maps variable names to metrics to use in your script.
|
||||
- `parent_bucket_path`, which is a path to a multi-bucket aggregation. The path can include single-bucket aggregations, but the last aggregation must be multi-bucket. For example, if you have a pipeline such as `agg1>agg2>agg3`, `agg1` and `agg2` are single-bucket aggregations, but `agg3` must be a multi-bucket aggregation.
|
||||
- `script`, which is the script that OpenSearch runs to evaluate whether to trigger any alerts.
|
||||
|
||||
For example, you might have a script that looks like the following:
|
||||
|
||||
```json
|
||||
{
|
||||
"buckets_path": {
|
||||
"count_var": "_count"
|
||||
},
|
||||
"parent_bucket_path": "composite_agg",
|
||||
"script": {
|
||||
"source": "params.count_var > 5"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
After mapping the `count_var` variable to the `_count` metric, you can use `count_var` in your script and reference `_count` data. Finally, `composite_agg` is a path to a multi-bucket aggregation.
|
||||
|
||||
### Anomaly detector
|
||||
|
||||
|
|
@ -264,11 +292,11 @@ Below are some variables you can include in your message using Mustache template
|
|||
|
||||
Variable | Data Type | Description
|
||||
:--- | :--- | :---
|
||||
`ctx.monitor` | JSON | Includes `ctx.monitor.name`, `ctx.monitor.type`, `ctx.monitor.enabled`, `ctx.monitor.enabled_time`, `ctx.monitor.schedule`, `ctx.monitor.inputs`, `triggers` and `ctx.monitor.last_update_time`.
|
||||
`ctx.monitor.user` | JSON | Includes information about the user who created the monitor. Includes `ctx.monitor.user.backend_roles` and `ctx.monitor.user.roles`, which are arrays that contain the backend roles and roles assigned to the user. See [alerting security]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/security/) for more information.
|
||||
`ctx.monitor` | Object | Includes `ctx.monitor.name`, `ctx.monitor.type`, `ctx.monitor.enabled`, `ctx.monitor.enabled_time`, `ctx.monitor.schedule`, `ctx.monitor.inputs`, `triggers` and `ctx.monitor.last_update_time`.
|
||||
`ctx.monitor.user` | Object | Includes information about the user who created the monitor. Includes `ctx.monitor.user.backend_roles` and `ctx.monitor.user.roles`, which are arrays that contain the backend roles and roles assigned to the user. See [alerting security]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/security/) for more information.
|
||||
`ctx.monitor.enabled` | Boolean | Whether the monitor is enabled.
|
||||
`ctx.monitor.enabled_time` | Milliseconds | Unix epoch time of when the monitor was last enabled.
|
||||
`ctx.monitor.schedule` | JSON | Contains a schedule of how often or when the monitor should run.
|
||||
`ctx.monitor.schedule` | Object | Contains a schedule of how often or when the monitor should run.
|
||||
`ctx.monitor.schedule.period.interval` | Integer | The interval at which the monitor runs.
|
||||
`ctx.monitor.schedule.period.unit` | String | The interval's unit of time.
|
||||
`ctx.monitor.inputs` | Array | An array that contains the indices and definition used to create the monitor.
|
||||
|
|
@ -282,7 +310,7 @@ Variable | Data Type | Description
|
|||
`ctx.trigger.id` | String | The trigger's ID.
|
||||
`ctx.trigger.name` | String | The trigger's name.
|
||||
`ctx.trigger.severity` | String | The trigger's severity.
|
||||
`ctx.trigger.condition`| JSON | Contains the Painless script used when creating the monitor.
|
||||
`ctx.trigger.condition`| Object | Contains the Painless script used when creating the monitor.
|
||||
`ctx.trigger.condition.script.source` | String | The language used to define the script. Must be painless.
|
||||
`ctx.trigger.condition.script.lang` | String | The script used to define the trigger.
|
||||
`ctx.trigger.actions`| Array | An array with one element that contains information about the action the monitor needs to trigger.
|
||||
|
|
@ -309,7 +337,13 @@ Variable | Data Type | Description
|
|||
`ctx.periodStart` | String | Unix timestamp for the beginning of the period during which the alert triggered. For example, if a monitor runs every ten minutes, a period might begin at 10:40 and end at 10:50.
|
||||
`ctx.periodEnd` | String | The end of the period during which the alert triggered.
|
||||
`ctx.error` | String | The error message if the trigger was unable to retrieve results or unable to evaluate the trigger, typically due to a compile error or null pointer exception. Null otherwise.
|
||||
`ctx.alert` | JSON | The current, active alert (if it exists). Includes `ctx.alert.id`, `ctx.alert.version`, and `ctx.alert.isAcknowledged`. Null if no alert is active.
|
||||
`ctx.alert` | Object | The current, active alert (if it exists). Includes `ctx.alert.id`, `ctx.alert.version`, and `ctx.alert.isAcknowledged`. Null if no alert is active. Only available with query-level monitors.
|
||||
`ctx.dedupedAlerts` | Object | Alerts that have already been triggered. OpenSearch keeps the existing alert to prevent the plugin from creating endless amounts of the same alerts. Only available with bucket-level monitors.
|
||||
`ctx.newAlerts` | Object | Newly created alerts. Only available with bucket-level monitors.
|
||||
`ctx.completedAlerts` | Object | Alerts that are no longer ongoing. Only available with bucket-level monitors.
|
||||
`bucket_keys` | String | Comma-separated list of the monitor's bucket key values. Available only for `ctx.dedupedAlerts`, `ctx.newAlerts`, and `ctx.completedAlerts`. Accessed through `ctx.dedupedAlerts[0].bucket_keys`.
|
||||
`parent_bucket_path` | String | The parent bucket path of the bucket that triggered the alert. Accessed through `ctx.dedupedAlerts[0].parent_bucket_path`.
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
|
@ -334,6 +368,7 @@ If you don't want to receive notifications for alerts, you don't have to add act
|
|||
```
|
||||
|
||||
In this case, the message content must conform to the `Content-Type` header in the [custom webhook](#create-destinations).
|
||||
1. If you're using a bucket-level monitor, you can choose whether the monitor should perform an action for each execution or for each alert.
|
||||
|
||||
1. (Optional) Use action throttling to limit the number of notifications you receive within a given span of time.
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ This list is extensive. We recommend using Ctrl/Cmd + F to find what you're look
|
|||
<tr>
|
||||
<td>CPU_Utilization
|
||||
</td>
|
||||
<td rowspan="18">ShardID, IndexName, Operation, ShardRole
|
||||
<td rowspan="20">ShardID, IndexName, Operation, ShardRole
|
||||
</td>
|
||||
<td>CPU usage ratio. CPU time (in milliseconds) used by the associated thread(s) in the past five seconds, divided by 5000 milliseconds.
|
||||
</td>
|
||||
|
|
@ -121,6 +121,18 @@ This list is extensive. We recommend using Ctrl/Cmd + F to find what you're look
|
|||
<td>The total number of times that the associated thread(s) blocked to enter or reenter a monitor (i.e. the number of times a thread has been in the blocked state).
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Thread_Waited_Time
|
||||
</td>
|
||||
<td>Average time (seconds) that the associated thread(s) waited to enter or reenter a monitor in WAITING or TIMED_WAITING state.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Thread_Waited_Event
|
||||
</td>
|
||||
<td>The total number of times that the associated thread(s) waited to enter or reenter a monitor (i.e. the number of times a thread has been in the WAITING or TIMED_WAITING state).
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ShardEvents
|
||||
</td>
|
||||
|
|
@ -315,6 +327,38 @@ This list is extensive. We recommend using Ctrl/Cmd + F to find what you're look
|
|||
<td>Estimated disk usage of the shard in bytes.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Indexing_Pressure_Current_Limits
|
||||
</td>
|
||||
<td rowspan="5">ShardID, IndexName, IndexingStage
|
||||
</td>
|
||||
<td>Total heap size (in bytes) that is available for utilization by a shard of an index in a particular indexing stage (Coordinating, Primary or Replica).
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Indexing_Pressure_Current_Bytes
|
||||
</td>
|
||||
<td>Total heap size (in bytes) occupied by a shard of an index in a particular indexing stage (Coordinating, Primary or Replica).
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Indexing_Pressure_Last_Successful_Timestamp
|
||||
</td>
|
||||
<td>Timestamp of a request that was successful for a shard of an index in a particular indexing stage (Coordinating, Primary or Replica).
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Indexing_Pressure_Rejection_Count
|
||||
</td>
|
||||
<td>Total rejections performed by OpenSearch for a shard of an index in a particular indexing stage (Coordinating, Primary or Replica).
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Indexing_Pressure_Average_Window_Throughput
|
||||
</td>
|
||||
<td>Average throughput of the last n requests (The value of n is determined by `shard_indexing_pressure.secondary_parameter.throughput.request_size_window` setting) for a shard of an index in a particular indexing stage (Coordinating, Primary or Replica).
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Latency
|
||||
</td>
|
||||
|
|
@ -424,7 +468,7 @@ This list is extensive. We recommend using Ctrl/Cmd + F to find what you're look
|
|||
</td>
|
||||
<td rowspan="5">Direction
|
||||
</td>
|
||||
<td>The total number of IPv4 datagrams transmitted/received from/by interfaces per second, including those transmitted or received in error
|
||||
<td>The total number of IPv4 datagrams transmitted/received from/by interfaces per second, including those transmitted or received in error.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
|
|
@ -454,7 +498,7 @@ This list is extensive. We recommend using Ctrl/Cmd + F to find what you're look
|
|||
<tr>
|
||||
<td>ThreadPool_QueueSize
|
||||
</td>
|
||||
<td rowspan="4">ThreadPoolType
|
||||
<td rowspan="6">ThreadPoolType
|
||||
</td>
|
||||
<td>The size of the task queue.
|
||||
</td>
|
||||
|
|
@ -477,10 +521,22 @@ This list is extensive. We recommend using Ctrl/Cmd + F to find what you're look
|
|||
<td>The approximate number of threads that are actively executing tasks.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ThreadPool_QueueLatency
|
||||
</td>
|
||||
<td>The latency of the task queue.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ThreadPool_QueueCapacity
|
||||
</td>
|
||||
<td>The current capacity of the task queue.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Master_PendingQueueSize
|
||||
</td>
|
||||
<td>N/A
|
||||
<td>Master_PendingTaskType
|
||||
</td>
|
||||
<td>The current number of pending tasks in the cluster state update thread. Each node has a cluster state update thread that submits cluster state update tasks (create index, update mapping, allocate shard, fail shard, etc.).
|
||||
</td>
|
||||
|
|
@ -533,6 +589,108 @@ This list is extensive. We recommend using Ctrl/Cmd + F to find what you're look
|
|||
<td>The time (milliseconds) that a master task has been executed.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Cache_MaxSize
|
||||
</td>
|
||||
<td>CacheType
|
||||
</td>
|
||||
<td>The max size of the cache in bytes.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>AdmissionControl_RejectionCount (WIP)
|
||||
</td>
|
||||
<td rowspan="3">ControllerName
|
||||
</td>
|
||||
<td>Total rejections performed by a Controller of Admission Control.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>AdmissionControl_CurrentValue (WIP)
|
||||
</td>
|
||||
<td>Current value for Controller of Admission Control.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>AdmissionControl_ThresholdValue (WIP)
|
||||
</td>
|
||||
<td>Threshold value for Controller of Admission Control.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Data_RetryingPendingTasksCount (WIP)
|
||||
</td>
|
||||
<td rowspan="2"> NodeID
|
||||
</td>
|
||||
<td>Number of throttled pending tasks on which data node is actively performing retries. It will be an absolute metric at that point of time.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Master_ThrottledPendingTasksCount (WIP)
|
||||
</td>
|
||||
<td>Sum of total pending tasks which got throttled by node (master node). It is a cumulative metric so look at the max aggregation.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Election_Term (WIP)
|
||||
</td>
|
||||
<td rowspan="5">N/A
|
||||
</td>
|
||||
<td>Monotonically increasing number with every master election.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>PublishClusterState_Latency (WIP)
|
||||
</td>
|
||||
<td>The time taken by quorum of nodes to publish new cluster state. This metric is available for current master.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>PublishClusterState_Failure (WIP)
|
||||
</td>
|
||||
<td>The number of times publish new cluster state action failed on master node.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ClusterApplierService_Latency (WIP)
|
||||
</td>
|
||||
<td>The time taken by each node to apply cluster state sent by master.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>ClusterApplierService_Failure (WIP)
|
||||
</td>
|
||||
<td>The number of times apply cluster state action failed on each node.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Shard_State (WIP)
|
||||
</td>
|
||||
<td>IndexName, NodeName, ShardType, ShardID
|
||||
</td>
|
||||
<td>The state of each shard - whether it is STARTED, UNASSIGNED, RELOCATING etc.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>LeaderCheck_Latency (WIP)
|
||||
</td>
|
||||
<td rowspan="4">WIP
|
||||
</td>
|
||||
<td rowspan="4">WIP
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>FollowerCheck_Failure (WIP)
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>LeaderCheck_Failure (WIP)
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>FollowerCheck_Latency (WIP)
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
|
|
@ -558,3 +716,4 @@ MasterTaskInsertOrder | The order in which the task was inserted (e.g. `3691`).
|
|||
MasterTaskPriority | Priority of the task (e.g. `URGENT`). OpenSearch executes higher priority tasks before lower priority ones, regardless of `insert_order`.
|
||||
MasterTaskType | `shard-started`, `create-index`, `delete-index`, `refresh-mapping`, `put-mapping`, `CleanupSnapshotRestoreState`, `Update snapshot state`
|
||||
MasterTaskMetadata | Metadata for the task (if any).
|
||||
CacheType | `Field_Data_Cache`, `Shard_Request_Cache`, `Node_Query_Cache`
|
||||
|
|
|
|||
|
|
@ -1,178 +0,0 @@
|
|||
---
|
||||
layout: default
|
||||
title: Configuration reference
|
||||
parent: Trace analytics
|
||||
nav_order: 25
|
||||
---
|
||||
|
||||
# Data Prepper configuration reference
|
||||
|
||||
This page lists all supported Data Prepper sources, buffers, preppers, and sinks, along with their associated options. For example configuration files, see [Data Prepper]({{site.url}}{{site.baseurl}}/monitoring-plugins/trace/data-prepper/).
|
||||
|
||||
|
||||
## Data Prepper server options
|
||||
|
||||
Option | Required | Description
|
||||
:--- | :--- | :---
|
||||
ssl | No | Boolean, indicating whether TLS should be used for server APIs. Defaults to true.
|
||||
keyStoreFilePath | No | String, path to a .jks or .p12 keystore file. Required if ssl is true.
|
||||
keyStorePassword | No | String, password for keystore. Optional, defaults to empty string.
|
||||
privateKeyPassword | No | String, password for private key within keystore. Optional, defaults to empty string.
|
||||
serverPort | No | Integer, port number to use for server APIs. Defaults to 4900
|
||||
|
||||
|
||||
## General pipeline options
|
||||
|
||||
Option | Required | Description
|
||||
:--- | :--- | :---
|
||||
workers | No | Integer, default 1. Essentially the number of application threads. As a starting point for your use case, try setting this value to the number of CPU cores on the machine.
|
||||
delay | No | Integer (milliseconds), default 3,000. How long workers wait between buffer read attempts.
|
||||
|
||||
|
||||
## Sources
|
||||
|
||||
Sources define where your data comes from.
|
||||
|
||||
|
||||
### otel_trace_source
|
||||
|
||||
Source for the OpenTelemetry Collector.
|
||||
|
||||
Option | Required | Description
|
||||
:--- | :--- | :---
|
||||
ssl | No | Boolean, whether to connect to the OpenTelemetry Collector over SSL.
|
||||
sslKeyCertChainFile | No | String, path to the security certificate (e.g. `"config/demo-data-prepper.crt"`.
|
||||
sslKeyFile | No | String, path to the security certificate key (e.g. `"config/demo-data-prepper.key"`).
|
||||
|
||||
|
||||
### file
|
||||
|
||||
Source for flat file input.
|
||||
|
||||
Option | Required | Description
|
||||
:--- | :--- | :---
|
||||
path | Yes | String, path to the input file (e.g. `logs/my-log.log`).
|
||||
|
||||
|
||||
### pipeline
|
||||
|
||||
Source for reading from another pipeline.
|
||||
|
||||
Option | Required | Description
|
||||
:--- | :--- | :---
|
||||
name | Yes | String, name of the pipeline to read from.
|
||||
|
||||
|
||||
### stdin
|
||||
|
||||
Source for console input. Can be useful for testing. No options.
|
||||
|
||||
|
||||
## Buffers
|
||||
|
||||
Buffers store data as it passes through the pipeline. If you implement a custom buffer, it can be memory-based (better performance) or disk-based (larger).
|
||||
|
||||
|
||||
### bounded_blocking
|
||||
|
||||
The default buffer. Memory-based.
|
||||
|
||||
Option | Required | Description
|
||||
:--- | :--- | :---
|
||||
buffer_size | No | Integer, default 512. The maximum number of records the buffer accepts.
|
||||
batch_size | No | Integer, default 8. The maximum number of records the buffer drains after each read.
|
||||
|
||||
|
||||
## Preppers
|
||||
|
||||
Preppers perform some action on your data: filter, transform, enrich, etc.
|
||||
|
||||
|
||||
### otel_trace_raw_prepper
|
||||
|
||||
Converts OpenTelemetry data to OpenSearch-compatible JSON documents.
|
||||
|
||||
Option | Required | Description
|
||||
:--- | :--- | :---
|
||||
root_span_flush_delay | No | Integer, representing the time interval in seconds to flush all the root spans in the prepper together with their descendants. Defaults to 30.
|
||||
trace_flush_interval | No | Integer, representing the time interval in seconds to flush all the descendant spans without any root span. Defaults to 180.
|
||||
|
||||
|
||||
### service_map_stateful
|
||||
|
||||
Uses OpenTelemetry data to create a distributed service map for visualization in OpenSearch Dashboards.
|
||||
|
||||
Option | Required | Description
|
||||
:--- | :--- | :---
|
||||
window_duration | No | Integer, representing the fixed time window in seconds to evaluate service-map relationships. Defaults to 180.
|
||||
|
||||
### peer_forwarder
|
||||
|
||||
Forwards ExportTraceServiceRequests via gRPC to other Data Prepper instances. Required for operating Data Prepper in a clustered deployment.
|
||||
|
||||
Option | Required | Description
|
||||
:--- | :--- | :---
|
||||
time_out | No | Integer, forwarded request timeout in seconds. Defaults to 3 seconds.
|
||||
span_agg_count | No | Integer, batch size for number of spans per request. Defaults to 48.
|
||||
discovery_mode | No | String, peer discovery mode to be used. Allowable values are `static` and `dns`. Defaults to `static`.
|
||||
static_endpoints | No | List, containing string endpoints of all Data Prepper instances.
|
||||
domain_name | No | String, single domain name to query DNS against. Typically used by creating multiple DNS A Records for the same domain.
|
||||
ssl | No | Boolean, indicating whether TLS should be used. Default is true.
|
||||
sslKeyCertChainFile | No | String, path to the security certificate
|
||||
|
||||
### string_converter
|
||||
|
||||
Converts strings to uppercase or lowercase. Mostly useful as an example if you want to develop your own prepper.
|
||||
|
||||
Option | Required | Description
|
||||
:--- | :--- | :---
|
||||
upper_case | No | Boolean, whether to convert to uppercase (`true`) or lowercase (`false`).
|
||||
|
||||
|
||||
## Sinks
|
||||
|
||||
Sinks define where Data Prepper writes your data to.
|
||||
|
||||
|
||||
### opensearch
|
||||
|
||||
Sink for an OpenSearch cluster.
|
||||
|
||||
Option | Required | Description
|
||||
:--- | :--- | :---
|
||||
hosts | Yes | List of OpenSearch hosts to write to (e.g. `["https://localhost:9200", "https://remote-cluster:9200"]`).
|
||||
cert | No | String, path to the security certificate (e.g. `"config/root-ca.pem"`) if the cluster uses the OpenSearch security plugin.
|
||||
username | No | String, username for HTTP basic authentication.
|
||||
password | No | String, password for HTTP basic authentication.
|
||||
aws_sigv4 | No | Boolean, whether to use IAM signing to connect to an Amazon ES cluster. For your access key, secret key, and optional session token, Data Prepper uses the default credential chain (environment variables, Java system properties, `~/.aws/credential`, etc.).
|
||||
aws_region | No | String, AWS region for the cluster (e.g. `"us-east-1"`) if you are connecting to Amazon ES.
|
||||
trace_analytics_raw | No | Boolean, default false. Whether to export as trace data to the `otel-v1-apm-span-*` index pattern (alias `otel-v1-apm-span`) for use with the Trace Analytics OpenSearch Dashboards plugin.
|
||||
trace_analytics_service_map | No | Boolean, default false. Whether to export as trace data to the `otel-v1-apm-service-map` index for use with the service map component of the Trace Analytics OpenSearch Dashboards plugin.
|
||||
index | No | String, name of the index to export to. Only required if you don't use the `trace_analytics_raw` or `trace_analytics_service_map` presets.
|
||||
template_file | No | String, the path to a JSON [index template]({{site.url}}{{site.baseurl}}/opensearch/index-templates/) file (e.g. `/your/local/template-file.json` if you do not use the `trace_analytics_raw` or `trace_analytics_service_map`. See [otel-v1-apm-span-index-template.json](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-plugins/opensearch/src/main/resources/otel-v1-apm-span-index-template.json) for an example.
|
||||
document_id_field | No | String, the field from the source data to use for the OpenSearch document ID (e.g. `"my-field"`) if you don't use the `trace_analytics_raw` or `trace_analytics_service_map` presets.
|
||||
dlq_file | No | String, the path to your preferred dead letter queue file (e.g. `/your/local/dlq-file`). Data Prepper writes to this file when it fails to index a document on the OpenSearch cluster.
|
||||
bulk_size | No | Integer (long), default 5. The maximum size (in MiB) of bulk requests to the OpenSearch cluster. Values below 0 indicate an unlimited size. If a single document exceeds the maximum bulk request size, Data Prepper sends it individually.
|
||||
|
||||
|
||||
### file
|
||||
|
||||
Sink for flat file output.
|
||||
|
||||
Option | Required | Description
|
||||
:--- | :--- | :---
|
||||
path | Yes | String, path for the output file (e.g. `logs/my-transformed-log.log`).
|
||||
|
||||
|
||||
### pipeline
|
||||
|
||||
Sink for writing to another pipeline.
|
||||
|
||||
Option | Required | Description
|
||||
:--- | :--- | :---
|
||||
name | Yes | String, name of the pipeline to write to.
|
||||
|
||||
|
||||
### stdout
|
||||
|
||||
Sink for console output. Can be useful for testing. No options.
|
||||
|
|
@ -1,135 +0,0 @@
|
|||
---
|
||||
layout: default
|
||||
title: Data Prepper
|
||||
parent: Trace analytics
|
||||
nav_order: 20
|
||||
---
|
||||
|
||||
# Data Prepper
|
||||
|
||||
Data Prepper is an independent component, not an OpenSearch plugin, that converts data for use with OpenSearch. It's not bundled with the all-in-one OpenSearch installation packages.
|
||||
|
||||
|
||||
## Install Data Prepper
|
||||
|
||||
To use the Docker image, pull it like any other image:
|
||||
|
||||
```bash
|
||||
docker pull opensearchproject/data-prepper:latest
|
||||
```
|
||||
|
||||
Otherwise, [download](https://opensearch.org/downloads.html) the appropriate archive for your operating system and unzip it.
|
||||
|
||||
|
||||
## Configure pipelines
|
||||
|
||||
To use Data Prepper, you define pipelines in a configuration YAML file. Each pipeline is a combination of a source, a buffer, zero or more preppers, and one or more sinks:
|
||||
|
||||
```yml
|
||||
sample-pipeline:
|
||||
workers: 4 # the number of workers
|
||||
delay: 100 # in milliseconds, how long workers wait between read attempts
|
||||
source:
|
||||
otel_trace_source:
|
||||
ssl: true
|
||||
sslKeyCertChainFile: "config/demo-data-prepper.crt"
|
||||
sslKeyFile: "config/demo-data-prepper.key"
|
||||
buffer:
|
||||
bounded_blocking:
|
||||
buffer_size: 1024 # max number of records the buffer accepts
|
||||
batch_size: 256 # max number of records the buffer drains after each read
|
||||
prepper:
|
||||
- otel_trace_raw_prepper:
|
||||
sink:
|
||||
- opensearch:
|
||||
hosts: ["https:localhost:9200"]
|
||||
cert: "config/root-ca.pem"
|
||||
username: "ta-user"
|
||||
password: "ta-password"
|
||||
trace_analytics_raw: true
|
||||
```
|
||||
|
||||
- Sources define where your data comes from. In this case, the source is the OpenTelemetry Collector (`otel_trace_source`) with some optional SSL settings.
|
||||
|
||||
- Buffers store data as it passes through the pipeline.
|
||||
|
||||
By default, Data Prepper uses its one and only buffer, the `bounded_blocking` buffer, so you can omit this section unless you developed a custom buffer or need to tune the buffer settings.
|
||||
|
||||
- Preppers perform some action on your data: filter, transform, enrich, etc.
|
||||
|
||||
You can have multiple preppers, which run sequentially from top to bottom, not in parallel. The `otel_trace_raw_prepper` prepper converts OpenTelemetry data into OpenSearch-compatible JSON documents.
|
||||
|
||||
- Sinks define where your data goes. In this case, the sink is an OpenSearch cluster.
|
||||
|
||||
Pipelines can act as the source for other pipelines. In the following example, a pipeline takes data from the OpenTelemetry Collector and uses two other pipelines as sinks:
|
||||
|
||||
```yml
|
||||
entry-pipeline:
|
||||
delay: "100"
|
||||
source:
|
||||
otel_trace_source:
|
||||
ssl: true
|
||||
sslKeyCertChainFile: "config/demo-data-prepper.crt"
|
||||
sslKeyFile: "config/demo-data-prepper.key"
|
||||
sink:
|
||||
- pipeline:
|
||||
name: "raw-pipeline"
|
||||
- pipeline:
|
||||
name: "service-map-pipeline"
|
||||
raw-pipeline:
|
||||
source:
|
||||
pipeline:
|
||||
name: "entry-pipeline"
|
||||
prepper:
|
||||
- otel_trace_raw_prepper:
|
||||
sink:
|
||||
- opensearch:
|
||||
hosts: ["https://localhost:9200" ]
|
||||
cert: "config/root-ca.pem"
|
||||
username: "ta-user"
|
||||
password: "ta-password"
|
||||
trace_analytics_raw: true
|
||||
service-map-pipeline:
|
||||
delay: "100"
|
||||
source:
|
||||
pipeline:
|
||||
name: "entry-pipeline"
|
||||
prepper:
|
||||
- service_map_stateful:
|
||||
sink:
|
||||
- opensearch:
|
||||
hosts: ["https://localhost:9200"]
|
||||
cert: "config/root-ca.pem"
|
||||
username: "ta-user"
|
||||
password: "ta-password"
|
||||
trace_analytics_service_map: true
|
||||
```
|
||||
|
||||
To learn more, see the [Data Prepper configuration reference]({{site.url}}{{site.baseurl}}/monitoring-plugins/trace/data-prepper-reference/).
|
||||
|
||||
## Configure the Data Prepper server
|
||||
Data Prepper itself provides administrative HTTP endpoints such as `/list` to list pipelines and `/metrics/prometheus` to provide Prometheus-compatible metrics data. The port which serves these endpoints, as well as TLS configuration, is specified by a separate YAML file. Example:
|
||||
|
||||
```yml
|
||||
ssl: true
|
||||
keyStoreFilePath: "/usr/share/data-prepper/keystore.jks"
|
||||
keyStorePassword: "password"
|
||||
privateKeyPassword: "other_password"
|
||||
serverPort: 1234
|
||||
```
|
||||
|
||||
## Start Data Prepper
|
||||
|
||||
**Docker**
|
||||
|
||||
```bash
|
||||
docker run --name data-prepper --expose 21890 -v /full/path/to/pipelines.yaml:/usr/share/data-prepper/pipelines.yaml -v /full/path/to/data-prepper-config.yaml:/usr/share/data-prepper/data-prepper-config.yaml opensearchproject/opensearch-data-prepper:latest
|
||||
```
|
||||
|
||||
**macOS and Linux**
|
||||
|
||||
```bash
|
||||
./data-prepper-tar-install.sh config/pipelines.yaml config/data-prepper-config.yaml
|
||||
```
|
||||
|
||||
For production workloads, you likely want to run Data Prepper on a dedicated machine, which makes connectivity a concern. Data Prepper uses port 21890 and must be able to connect to both the OpenTelemetry Collector and the OpenSearch cluster. In the [sample applications](https://github.com/opensearch-project/Data-Prepper/tree/main/examples), you can see that all components use the same Docker network and expose the appropriate ports.
|
||||
|
|
@ -0,0 +1,231 @@
|
|||
---
|
||||
layout: default
|
||||
title: Configuration reference
|
||||
parent: Data Prepper
|
||||
nav_order: 3
|
||||
---
|
||||
|
||||
# Data Prepper configuration reference
|
||||
|
||||
This page lists all supported Data Prepper server, sources, buffers, preppers, and sinks, along with their associated options. For example configuration files, see [Data Prepper]({{site.url}}{{site.baseurl}}/observability/data-prepper/pipelines/).
|
||||
|
||||
## Data Prepper server options
|
||||
|
||||
Option | Required | Type | Description
|
||||
:--- | :--- | :--- | :---
|
||||
ssl | No | Boolean | Indicates whether TLS should be used for server APIs. Defaults to true.
|
||||
keyStoreFilePath | No | String | Path to a .jks or .p12 keystore file. Required if ssl is true.
|
||||
keyStorePassword | No | String | Password for keystore. Optional, defaults to empty string.
|
||||
privateKeyPassword | No | String | Password for private key within keystore. Optional, defaults to empty string.
|
||||
serverPort | No | Integer | Port number to use for server APIs. Defaults to 4900
|
||||
metricRegistries | No | List | Metrics registries for publishing the generated metrics. Currently supports Prometheus and CloudWatch. Defaults to Prometheus.
|
||||
|
||||
## General pipeline options
|
||||
|
||||
Option | Required | Type | Description
|
||||
:--- | :--- | :--- | :---
|
||||
workers | No | Integer | Essentially the number of application threads. As a starting point for your use case, try setting this value to the number of CPU cores on the machine. Default is 1.
|
||||
delay | No | Integer | Amount of time in milliseconds workers wait between buffer read attempts. Default is 3,000.
|
||||
|
||||
|
||||
## Sources
|
||||
|
||||
Sources define where your data comes from.
|
||||
|
||||
|
||||
### otel_trace_source
|
||||
|
||||
Source for the OpenTelemetry Collector.
|
||||
|
||||
Option | Required | Type | Description
|
||||
:--- | :--- | :--- | :---
|
||||
port | No | Integer | The port OTel trace source is running on. Default is `21890`.
|
||||
request_timeout | No | Integer | The request timeout in milliseconds. Default is `10_000`.
|
||||
health_check_service | No | Boolean | Enables a gRPC health check service under `grpc.health.v1/Health/Check`. Default is `false`.
|
||||
proto_reflection_service | No | Boolean | Enables a reflection service for Protobuf services (see [gRPC reflection](https://github.com/grpc/grpc/blob/master/doc/server-reflection.md) and [gRPC Server Reflection Tutorial](https://github.com/grpc/grpc-java/blob/master/documentation/server-reflection-tutorial.md) docs). Default is `false`.
|
||||
unframed_requests | No | Boolean | Enable requests not framed using the gRPC wire protocol.
|
||||
thread_count | No | Integer | The number of threads to keep in the ScheduledThreadPool. Default is `200`.
|
||||
max_connection_count | No | Integer | The maximum allowed number of open connections. Default is `500`.
|
||||
ssl | No | Boolea | Enables connections to the OTel source port over TLS/SSL. Defaults to `true`.
|
||||
sslKeyCertChainFile | Conditionally | String | File-system path or AWS S3 path to the security certificate (e.g. `"config/demo-data-prepper.crt"` or `"s3://my-secrets-bucket/demo-data-prepper.crt"`). Required if ssl is set to `true`.
|
||||
sslKeyFile | Conditionally | String | File-system path or AWS S3 path to the security key (e.g. `"config/demo-data-prepper.key"` or `"s3://my-secrets-bucket/demo-data-prepper.key"`). Required if ssl is set to `true`.
|
||||
useAcmCertForSSL | No | Boolean, enables TLS/SSL using certificate and private key from AWS Certificate Manager (ACM). Default is `false`.
|
||||
acmCertificateArn | Conditionally | String | Represents the ACM certificate ARN. ACM certificate take preference over S3 or local file system certificate. Required if `useAcmCertForSSL` is set to `true`.
|
||||
awsRegion | Conditionally | String | Represents the AWS region to use ACM or S3. Required if `useAcmCertForSSL` is set to `true` or `sslKeyCertChainFile` and `sslKeyFile` are AWS S3 paths.
|
||||
authentication | No | Object| An authentication configuration. By default, this runs an unauthenticated server. This uses pluggable authentication for HTTPS. To use basic authentication, define the `http_basic` plugin with a `username` and `password`. To provide customer authentication use or create a plugin which implements: [GrpcAuthenticationProvider](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-plugins/armeria-common/src/main/java/com/amazon/dataprepper/armeria/authentication/GrpcAuthenticationProvider.java).
|
||||
|
||||
### http_source
|
||||
|
||||
This is a source plugin that supports HTTP protocol. Currently ONLY support Json UTF-8 codec for incoming request, e.g. `[{"key1": "value1"}, {"key2": "value2"}]`.
|
||||
|
||||
Option | Required | Type | Description
|
||||
:--- | :--- | :--- | :---
|
||||
port | No | Integer | The port the source is running on. Default is `2021`. Valid options are between `0` and `65535`.
|
||||
request_timeout | No | Integer | The request timeout in millis. Default is `10_000`.
|
||||
thread_count | No | Integer | The number of threads to keep in the ScheduledThreadPool. Default is `200`.
|
||||
max_connection_count | No | Integer | The maximum allowed number of open connections. Default is `500`.
|
||||
max_pending_requests | No | Integer | The maximum number of allowed tasks in ScheduledThreadPool work queue. Default is `1024`.
|
||||
authentication | No | Object | An authentication configuration. By default, this runs an unauthenticated server. This uses pluggable authentication for HTTPS. To use basic authentication define the `http_basic` plugin with a `username` and `password`. To provide customer authentication use or create a plugin which implements: [ArmeriaHttpAuthenticationProvider](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-plugins/armeria-common/src/main/java/com/amazon/dataprepper/armeria/authentication/ArmeriaHttpAuthenticationProvider.java).
|
||||
|
||||
### file
|
||||
|
||||
Source for flat file input.
|
||||
|
||||
Option | Required | Type | Description
|
||||
:--- | :--- | :--- | :---
|
||||
path | Yes | String | Path to the input file (e.g. `logs/my-log.log`).
|
||||
format | No | String | Format of each line in the file. Valid options are `json` or `plain`. Default is `plain`.
|
||||
record_type | No | String | The record type that will be stored. Valid options are `string` or `event`. Default is `string`. If you would like to use the file source for log analytics use cases like grok, set this option to `event`.
|
||||
|
||||
### pipeline
|
||||
|
||||
Source for reading from another pipeline.
|
||||
|
||||
Option | Required | Type | Description
|
||||
:--- | :--- | :--- | :---
|
||||
name | Yes | String | Name of the pipeline to read from.
|
||||
|
||||
|
||||
### stdin
|
||||
|
||||
Source for console input. Can be useful for testing. No options.
|
||||
|
||||
|
||||
## Buffers
|
||||
|
||||
Buffers store data as it passes through the pipeline. If you implement a custom buffer, it can be memory-based (better performance) or disk-based (larger).
|
||||
|
||||
|
||||
### bounded_blocking
|
||||
|
||||
The default buffer. Memory-based.
|
||||
|
||||
Option | Required | Type | Description
|
||||
:--- | :--- | :--- | :---
|
||||
buffer_size | No | Integer | The maximum number of records the buffer accepts. Default is 512.
|
||||
batch_size | No | Integer | The maximum number of records the buffer drains after each read. Default is 8.
|
||||
|
||||
|
||||
## Preppers
|
||||
|
||||
Preppers perform some action on your data: filter, transform, enrich, etc.
|
||||
|
||||
|
||||
### otel_trace_raw_prepper
|
||||
|
||||
Converts OpenTelemetry data to OpenSearch-compatible JSON documents.
|
||||
|
||||
Option | Required | Type | Description
|
||||
:--- | :--- | :--- | :---
|
||||
root_span_flush_delay | No | Integer | Represents the time interval in seconds to flush all the root spans in the prepper together with their descendants. Default is 30.
|
||||
trace_flush_interval | No | Integer | Represents the time interval in seconds to flush all the descendant spans without any root span. Default is 180.
|
||||
|
||||
|
||||
### service_map_stateful
|
||||
|
||||
Uses OpenTelemetry data to create a distributed service map for visualization in OpenSearch Dashboards.
|
||||
|
||||
Option | Required | Type | Description
|
||||
:--- | :--- | :--- | :---
|
||||
window_duration | No | Integer | Represents the fixed time window in seconds to evaluate service-map relationships. Default is 180.
|
||||
|
||||
### peer_forwarder
|
||||
|
||||
Forwards ExportTraceServiceRequests via gRPC to other Data Prepper instances. Required for operating Data Prepper in a clustered deployment.
|
||||
|
||||
Option | Required | Type | Description
|
||||
:--- | :--- | :--- | :---
|
||||
time_out | No | Integer | Forwarded request timeout in seconds. Defaults to 3 seconds.
|
||||
span_agg_count | No | Integer | Batch size for number of spans per request. Defaults to 48.
|
||||
target_port | No | Integer | The destination port to forward requests to. Defaults to `21890`.
|
||||
discovery_mode | No | String | Peer discovery mode to be used. Allowable values are `static`, `dns`, and `aws_cloud_map`. Defaults to `static`.
|
||||
static_endpoints | No | List | List containing string endpoints of all Data Prepper instances.
|
||||
domain_name | No | String | Single domain name to query DNS against. Typically used by creating multiple DNS A Records for the same domain.
|
||||
ssl | No | Boolean | Indicates whether TLS should be used. Default is true.
|
||||
awsCloudMapNamespaceName | Conditionally | String | Name of your CloudMap Namespace. Required if `discovery_mode` is set to `aws_cloud_map`.
|
||||
awsCloudMapServiceName | Conditionally | String | Service name within your CloudMap Namespace. Required if `discovery_mode` is set to `aws_cloud_map`.
|
||||
sslKeyCertChainFile | Conditionally | String | Represents the SSL certificate chain file path or AWS S3 path. S3 path example `s3://<bucketName>/<path>`. Required if `ssl` is set to `true`.
|
||||
useAcmCertForSSL | No | Boolean | Enables TLS/SSL using certificate and private key from AWS Certificate Manager (ACM). Default is `false`.
|
||||
awsRegion | Conditionally | String | Represents the AWS region to use ACM, S3, or CloudMap. Required if `useAcmCertForSSL` is set to `true` or `sslKeyCertChainFile` and `sslKeyFile` are AWS S3 paths.
|
||||
acmCertificateArn | Conditionally | String | Represents the ACM certificate ARN. ACM certificate take preference over S3 or local file system certificate. Required if `useAcmCertForSSL` is set to `true`.
|
||||
|
||||
### string_converter
|
||||
|
||||
Converts strings to uppercase or lowercase. Mostly useful as an example if you want to develop your own prepper.
|
||||
|
||||
Option | Required | Type | Description
|
||||
:--- | :--- | :--- | :---
|
||||
upper_case | No | Boolean | Whether to convert to uppercase (`true`) or lowercase (`false`).
|
||||
|
||||
### grok_prepper
|
||||
|
||||
Takes unstructured data and utilizes pattern matching to structure and extract important keys and make data more structured and queryable.
|
||||
|
||||
Option | Required | Type | Description
|
||||
:--- | :--- | :--- | :---
|
||||
match | No | Map | Specifies which keys to match specific patterns against. Default is an empty body.
|
||||
keep_empty_captures | No | Boolean | Enables preserving `null` captures. Default value is `false`.
|
||||
named_captures_only | No | Boolean | enables whether to keep only named captures. Default value is `true`.
|
||||
break_on_match | No | Boolean | Specifies wether to match all patterns or stop once the first successful match is found. Default is `true`.
|
||||
keys_to_overwrite | No | List | Specifies which existing keys are to be overwritten if there is a capture with the same key value. Default is `[]`.
|
||||
pattern_definitions | No | Map | Allows for custom pattern use inline. Default value is an empty body.
|
||||
patterns_directories | No | List | Specifies the path of directories that contain customer pattern files. Default value is an empty list.
|
||||
pattern_files_glob | No | String | Specifies which pattern files to use from the directories specified for `pattern_directories`. Default is `*`.
|
||||
target_key | No | String | Specifies a parent level key to store all captures. Default value is `null`.
|
||||
timeout_millis | No | Integer | Maximum amount of time that should take place for the matching. Setting to `0` disables the timeout. Default value is `30,000`.
|
||||
|
||||
## Sinks
|
||||
|
||||
Sinks define where Data Prepper writes your data to.
|
||||
|
||||
|
||||
### OpenSearch
|
||||
|
||||
Sink for an OpenSearch cluster.
|
||||
|
||||
Option | Required | Type | Description
|
||||
:--- | :--- | :--- | :---
|
||||
hosts | Yes | List | List of OpenSearch hosts to write to (e.g. `["https://localhost:9200", "https://remote-cluster:9200"]`).
|
||||
cert | No | String | Path to the security certificate (e.g. `"config/root-ca.pem"`) if the cluster uses the OpenSearch security plugin.
|
||||
username | No | String | Username for HTTP basic authentication.
|
||||
password | No | String | Password for HTTP basic authentication.
|
||||
aws_sigv4 | No | Boolean | default false. Whether to use IAM signing to connect to an Amazon OpenSearch Service domain. For your access key, secret key, and optional session token, Data Prepper uses the default credential chain (environment variables, Java system properties, `~/.aws/credential`, etc.).
|
||||
aws_region | No | String | AWS region (e.g. `"us-east-1"`) for the domain if you are connecting to Amazon OpenSearch Service.
|
||||
aws_sts_role_arn | No | String | IAM role which the sink plugin assumes to sign request to Amazon OpenSearch Service. If not provided the plugin uses the default credentials.
|
||||
socket_timeout | No | Integer | the timeout in milliseconds for waiting for data (or, put differently, a maximum period inactivity between two consecutive data packets). A timeout value of zero is interpreted as an infinite timeout. If this timeout value is either negative or not set, the underlying Apache HttpClient would rely on operating system settings for managing socket timeouts.
|
||||
connect_timeout | No | Integer | The timeout in milliseconds used when requesting a connection from the connection manager. A timeout value of zero is interpreted as an infinite timeout. If this timeout value is either negative or not set, the underlying Apache HttpClient would rely on operating system settings for managing connection timeouts.
|
||||
insecure | No | Boolean | Whether to verify SSL certificates. If set to true, CA certificate verification is disabled and insecure HTTP requests are sent instead. Default is false.
|
||||
proxy | No | String | The address of a [forward HTTP proxy server](https://en.wikipedia.org/wiki/Proxy_server). The format is "<host name or IP>:<port>". Examples: "example.com:8100", "http://example.com:8100", "112.112.112.112:8100". Port number cannot be omitted.
|
||||
trace_analytics_raw | No | Boolean | Deprecated in favor of `index_type`. Whether to export as trace data to the `otel-v1-apm-span-*` index pattern (alias `otel-v1-apm-span`) for use with the Trace Analytics OpenSearch Dashboards plugin. Default is false.
|
||||
trace_analytics_service_map | No | Boolean | Deprecated in favor of `index_type`. Whether to export as trace data to the `otel-v1-apm-service-map` index for use with the service map component of the Trace Analytics OpenSearch Dashboards plugin. | Default is false.
|
||||
index | No | String | Name of the index to export to. Only required if you don't use the `trace-analytics-raw` or `trace-analytics-service-map` presets. In other words, this parameter is applicable and required only if index_type is explicitly `custom` or defaults to `custom`.
|
||||
index_type | No | String | This index type instructs the Sink plugin what type of data it is handling. Valid values: `custom`, `trace-analytics-raw`, `trace-analytics-service-map`. Default is `custom`.
|
||||
template_file | No | String | Path to a JSON [index template]({{site.url}}{{site.baseurl}}/opensearch/index-templates/) file (e.g. `/your/local/template-file.json` if you do not use the `trace_analytics_raw` or `trace_analytics_service_map`.) See [otel-v1-apm-span-index-template.json](https://github.com/opensearch-project/data-prepper/blob/main/data-prepper-plugins/opensearch/src/main/resources/otel-v1-apm-span-index-template.json) for an example.
|
||||
document_id_field | No | String | The field from the source data to use for the OpenSearch document ID (e.g. `"my-field"`) if you don't use the `trace_analytics_raw` or `trace_analytics_service_map` presets.
|
||||
dlq_file | No | String | The path to your preferred dead letter queue file (e.g. `/your/local/dlq-file`). Data Prepper writes to this file when it fails to index a document on the OpenSearch cluster.
|
||||
bulk_size | No | Integer (long) | The maximum size (in MiB) of bulk requests to the OpenSearch cluster. Values below 0 indicate an unlimited size. If a single document exceeds the maximum bulk request size, Data Prepper sends it individually. Default is 5.
|
||||
ism_policy_file | No | String | The absolute file path for an ISM (Index State Management) policy JSON file. This policy file is effective only when there is no built-in policy file for the index type. For example, `custom` index type is currently the only one without a built-in policy file, thus it would use the policy file here if it's provided through this parameter. For more information, see [ISM policies]({{site.url}}{{site.baseurl}}/im-plugin/ism/policies/).
|
||||
number_of_shards | No | Integer | The number of primary shards that an index should have on the destination OpenSearch server. This parameter is effective only when `template_file` is either explicitly provided in Sink configuration or built-in. If this parameter is set, it would override the value in index template file. For more information, see [create index]({{site.url}}{{site.baseurl}}/opensearch/rest-api/index-apis/create-index/).
|
||||
number_of_replicas | No | Integer | The number of replica shards each primary shard should have on the destination OpenSearch server. For example, if you have 4 primary shards and set number_of_replicas to 3, the index has 12 replica shards. This parameter is effective only when `template_file` is either explicitly provided in Sink configuration or built-in. If this parameter is set, it would override the value in index template file. For more information, see [create index]({{site.url}}{{site.baseurl}}/opensearch/rest-api/index-apis/create-index/).
|
||||
|
||||
### file
|
||||
|
||||
Sink for flat file output.
|
||||
|
||||
Option | Required | Type | Description
|
||||
:--- | :--- | :--- | :---
|
||||
path | Yes | String | Path for the output file (e.g. `logs/my-transformed-log.log`).
|
||||
|
||||
|
||||
### pipeline
|
||||
|
||||
Sink for writing to another pipeline.
|
||||
|
||||
Option | Required | Type | Description
|
||||
:--- | :--- | :--- | :---
|
||||
name | Yes | String | Name of the pipeline to write to.
|
||||
|
||||
|
||||
### stdout
|
||||
|
||||
Sink for console output. Can be useful for testing. No options.
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
---
|
||||
layout: default
|
||||
title: Get Started
|
||||
parent: Data Prepper
|
||||
nav_order: 1
|
||||
---
|
||||
|
||||
# Get started with Data Prepper
|
||||
|
||||
Data Prepper is an independent component, not an OpenSearch plugin, that converts data for use with OpenSearch. It's not bundled with the all-in-one OpenSearch installation packages.
|
||||
|
||||
## 1. Install Data Prepper
|
||||
|
||||
To use the Docker image, pull it like any other image:
|
||||
|
||||
```bash
|
||||
docker pull opensearchproject/data-prepper:latest
|
||||
```
|
||||
|
||||
## 2. Define a pipeline
|
||||
|
||||
Create a Data Prepper pipeline file, `pipelines.yaml`, with the following configuration:
|
||||
|
||||
```yml
|
||||
simple-sample-pipeline:
|
||||
workers: 2
|
||||
delay: "5000"
|
||||
source:
|
||||
random:
|
||||
sink:
|
||||
- stdout:
|
||||
```
|
||||
|
||||
## 3. Start Data Prepper
|
||||
|
||||
Run the following command with your pipeline configuration YAML.
|
||||
|
||||
```bash
|
||||
docker run --name data-prepper \
|
||||
-v /full/path/to/pipelines.yaml:/usr/share/data-prepper/pipelines.yaml \
|
||||
opensearchproject/opensearch-data-prepper:latest
|
||||
```
|
||||
|
||||
This sample pipeline configuration above demonstrates a simple pipeline with a source (`random`) sending data to a sink (`stdout`). For more examples and details on more advanced pipeline configurations, see [Pipelines]({{site.url}}{{site.baseurl}}/observability/data-prepper/pipelines).
|
||||
|
||||
After starting Data Prepper, you should see log output and some UUIDs after a few seconds:
|
||||
|
||||
```yml
|
||||
2021-09-30T20:19:44,147 [main] INFO com.amazon.dataprepper.pipeline.server.DataPrepperServer - Data Prepper server running at :4900
|
||||
2021-09-30T20:19:44,681 [random-source-pool-0] INFO com.amazon.dataprepper.plugins.source.RandomStringSource - Writing to buffer
|
||||
2021-09-30T20:19:45,183 [random-source-pool-0] INFO com.amazon.dataprepper.plugins.source.RandomStringSource - Writing to buffer
|
||||
2021-09-30T20:19:45,687 [random-source-pool-0] INFO com.amazon.dataprepper.plugins.source.RandomStringSource - Writing to buffer
|
||||
2021-09-30T20:19:46,191 [random-source-pool-0] INFO com.amazon.dataprepper.plugins.source.RandomStringSource - Writing to buffer
|
||||
2021-09-30T20:19:46,694 [random-source-pool-0] INFO com.amazon.dataprepper.plugins.source.RandomStringSource - Writing to buffer
|
||||
2021-09-30T20:19:47,200 [random-source-pool-0] INFO com.amazon.dataprepper.plugins.source.RandomStringSource - Writing to buffer
|
||||
2021-09-30T20:19:49,181 [simple-test-pipeline-processor-worker-1-thread-1] INFO com.amazon.dataprepper.pipeline.ProcessWorker - simple-test-pipeline Worker: Processing 6 records from buffer
|
||||
07dc0d37-da2c-447e-a8df-64792095fb72
|
||||
5ac9b10a-1d21-4306-851a-6fb12f797010
|
||||
99040c79-e97b-4f1d-a70b-409286f2a671
|
||||
5319a842-c028-4c17-a613-3ef101bd2bdd
|
||||
e51e700e-5cab-4f6d-879a-1c3235a77d18
|
||||
b4ed2d7e-cf9c-4e9d-967c-b18e8af35c90
|
||||
```
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
---
|
||||
layout: default
|
||||
title: Data Prepper
|
||||
nav_order: 80
|
||||
has_children: true
|
||||
has_toc: false
|
||||
---
|
||||
|
||||
# Data Prepper
|
||||
|
||||
Data Prepper is a server side data collector capable of filtering, enriching, transforming, normalizing and aggregating data for downstream analytics and visualization.
|
||||
|
||||
Data Prepper lets users build custom pipelines to improve the operational view of applications. Two common uses for Data Prepper are trace and log analytics. [Trace analytics]({{site.url}}{{site.baseurl}}/observability/trace/index/) can help you visualize the flow of events and identify performance problems, and [log analytics]({{site.url}}{{site.baseurl}}/observability/log-analytics/) can improve searching, analyzing and provide insights into your application.
|
||||
|
||||
To get started building your own custom pipelines with Data Prepper, see the [Get Started]({{site.url}}{{site.baseurl}}/observability/data-prepper/get-started/) guide.
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
---
|
||||
layout: default
|
||||
title: Pipelines
|
||||
parent: Data Prepper
|
||||
nav_order: 2
|
||||
---
|
||||
|
||||
# Pipelines
|
||||
|
||||
|
||||
|
||||
To use Data Prepper, you define pipelines in a configuration YAML file. Each pipeline is a combination of a source, a buffer, zero or more preppers, and one or more sinks. For example:
|
||||
|
||||
```yml
|
||||
simple-sample-pipeline:
|
||||
workers: 2 # the number of workers
|
||||
delay: 5000 # in milliseconds, how long workers wait between read attempts
|
||||
source:
|
||||
random:
|
||||
buffer:
|
||||
bounded_blocking:
|
||||
buffer_size: 1024 # max number of records the buffer accepts
|
||||
batch_size: 256 # max number of records the buffer drains after each read
|
||||
processor:
|
||||
- string_converter:
|
||||
upper_case: true
|
||||
sink:
|
||||
- stdout:
|
||||
```
|
||||
|
||||
- Sources define where your data comes from. In this case, the source is a random UUID generator (`random`).
|
||||
|
||||
- Buffers store data as it passes through the pipeline.
|
||||
|
||||
By default, Data Prepper uses its one and only buffer, the `bounded_blocking` buffer, so you can omit this section unless you developed a custom buffer or need to tune the buffer settings.
|
||||
|
||||
- Preppers perform some action on your data: filter, transform, enrich, etc.
|
||||
|
||||
You can have multiple preppers, which run sequentially from top to bottom, not in parallel. The `string_converter` prepper transform the strings by making them uppercase.
|
||||
|
||||
- Sinks define where your data goes. In this case, the sink is stdout.
|
||||
|
||||
## Examples
|
||||
|
||||
This section provides some pipeline examples that you can use to start creating your own pipelines. For more information, see [Data Prepper configuration reference]({{site.url}}{{site.baseurl}}/observability/data-prepper/data-prepper-reference/) guide.
|
||||
|
||||
The Data Prepper repository has several [sample applications](https://github.com/opensearch-project/data-prepper/tree/main/examples) to help you get started.
|
||||
|
||||
### Log ingestion pipeline
|
||||
|
||||
The following example demonstrates how to use HTTP source and Grok prepper plugins to process unstructured log data.
|
||||
|
||||
```yml
|
||||
log-pipeline:
|
||||
source:
|
||||
http:
|
||||
ssl: false
|
||||
processor:
|
||||
- grok:
|
||||
match:
|
||||
log: [ "%{COMMONAPACHELOG}" ]
|
||||
sink:
|
||||
- opensearch:
|
||||
hosts: [ "https://opensearch:9200" ]
|
||||
insecure: true
|
||||
username: admin
|
||||
password: admin
|
||||
index: apache_logs
|
||||
```
|
||||
|
||||
This example uses weak security. We strongly recommend securing all plugins which open external ports in production environments.
|
||||
{: .note}
|
||||
|
||||
### Trace Analytics pipeline
|
||||
|
||||
The following example demonstrates how to build a pipeline that supports the [Trace Analytics OpenSearch Dashboards plugin]({{site.url}}{{site.baseurl}}/observability/trace/ta-dashboards/). This pipeline takes data from the OpenTelemetry Collector and uses two other pipelines as sinks. These two separate pipelines index trace and the service map documents for the dashboard plugin.
|
||||
|
||||
```yml
|
||||
entry-pipeline:
|
||||
delay: "100"
|
||||
source:
|
||||
otel_trace_source:
|
||||
ssl: false
|
||||
sink:
|
||||
- pipeline:
|
||||
name: "raw-pipeline"
|
||||
- pipeline:
|
||||
name: "service-map-pipeline"
|
||||
raw-pipeline:
|
||||
source:
|
||||
pipeline:
|
||||
name: "entry-pipeline"
|
||||
prepper:
|
||||
- otel_trace_raw_prepper:
|
||||
sink:
|
||||
- opensearch:
|
||||
hosts: ["https://localhost:9200"]
|
||||
insecure: true
|
||||
username: admin
|
||||
password: admin
|
||||
trace_analytics_raw: true
|
||||
service-map-pipeline:
|
||||
delay: "100"
|
||||
source:
|
||||
pipeline:
|
||||
name: "entry-pipeline"
|
||||
prepper:
|
||||
- service_map_stateful:
|
||||
sink:
|
||||
- opensearch:
|
||||
hosts: ["https://localhost:9200"]
|
||||
insecure: true
|
||||
username: admin
|
||||
password: admin
|
||||
trace_analytics_service_map: true
|
||||
```
|
||||
|
||||
## Migrating from Logstash
|
||||
|
||||
Data Prepper supports Logstash configuration files for a limited set of plugins. Simply use the logstash config to run Data Prepper.
|
||||
|
||||
```bash
|
||||
docker run --name data-prepper \
|
||||
-v /full/path/to/logstash.conf:/usr/share/data-prepper/pipelines.conf \
|
||||
opensearchproject/opensearch-data-prepper:latest
|
||||
```
|
||||
|
||||
This feature is limited by feature parity of Data Prepper. As of Data Prepper 1.2 release, the following plugins from the Logstash configuration are supported:
|
||||
|
||||
- HTTP Input plugin
|
||||
- Grok Filter plugin
|
||||
- Elasticsearch Output plugin
|
||||
- Amazon Elasticsearch Output plugin
|
||||
|
||||
## Configure the Data Prepper server
|
||||
|
||||
Data Prepper itself provides administrative HTTP endpoints such as `/list` to list pipelines and `/metrics/prometheus` to provide Prometheus-compatible metrics data. The port that has these endpoints has a TLS configuration and is specified by a separate YAML file. By default, these endpoints are secured by Data Prepper docker images. We strongly recommend providing your own configuration file for securing production environments. Here is an example `data-prepper-config.yaml`:
|
||||
|
||||
```yml
|
||||
ssl: true
|
||||
keyStoreFilePath: "/usr/share/data-prepper/keystore.jks"
|
||||
keyStorePassword: "password"
|
||||
privateKeyPassword: "other_password"
|
||||
serverPort: 1234
|
||||
```
|
||||
|
||||
To configure the Data Prepper server, run Data Prepper with the additional yaml file.
|
||||
|
||||
```bash
|
||||
docker run --name data-prepper -v /full/path/to/pipelines.yaml:/usr/share/data-prepper/pipelines.yaml \
|
||||
/full/path/to/data-prepper-config.yaml:/usr/share/data-prepper/data-prepper-config.yaml \
|
||||
opensearchproject/opensearch-data-prepper:latest
|
||||
````
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
---
|
||||
layout: default
|
||||
title: Event analytics
|
||||
nav_order: 10
|
||||
---
|
||||
|
||||
# Event analytics
|
||||
|
||||
Event analytics in observability is where you can use [Piped Processing Language]({{site.url}}{{site.baseurl}}/observability/ppl/index) (PPL) queries to build and view different visualizations of your data.
|
||||
|
||||
## Get started with event analytics
|
||||
|
||||
To get started, choose **Observability** in OpenSearch Dashboards, and then choose **Event analytics**. If you want to start exploring without adding any of your own data, choose **Add sample Events Data**, and Dashboards adds some sample visualizations you can interact with.
|
||||
|
||||
## Build a query
|
||||
|
||||
To generate custom visualizations, you must first specify a PPL query. OpenSearch Dashboards then automatically creates a visualization based on the results of your query.
|
||||
|
||||
For example, the following PPL query returns a count of how many host addresses are currently in your data.
|
||||
|
||||
```
|
||||
source = opensearch_dashboards_sample_data_logs | fields host | stats count()
|
||||
```
|
||||
|
||||
By default, Dashboards shows results from the last 15 minutes of your data. To see data from a different timeframe, use the date and time selector.
|
||||
|
||||
For more information about building PPL queries, see [Piped Processing Language]({{site.url}}{{site.baseurl}}/observability/ppl/index).
|
||||
|
||||
## Save a visualization
|
||||
|
||||
After Dashboards generates a visualization, you must save it if you want to return to it at a later time or if you want to add it to an [operational panel]({{site.url}}{{site.baseurl}}/observability/operational-panels).
|
||||
|
||||
To save a visualization, expand the save dropdown menu next to **Run**, enter a name for your visualization, then choose **Save**. You can reopen any saved visualizations on the event analytics page.
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
---
|
||||
layout: default
|
||||
title: About Observability
|
||||
nav_order: 1
|
||||
has_children: false
|
||||
redirect_from:
|
||||
- /observability/
|
||||
- /observability/
|
||||
---
|
||||
|
||||
# About Observability
|
||||
OpenSearch Dashboards
|
||||
{: .label .label-yellow :}
|
||||
|
||||
Observability is collection of plugins and applications that let you visualize data-driven events by using Piped Processing Language to explore, discover, and query data stored in OpenSearch.
|
||||
|
||||
Your experience of exploring data might differ, but if you're new to exploring data to create visualizations, we recommend trying a workflow like the following:
|
||||
|
||||
1. Explore data over a certain timeframe using [Piped Processing Language]({{site.url}}{{site.baseurl}}/observability/ppl/index).
|
||||
2. Use [event analytics]({{site.url}}{{site.baseurl}}/observability/event-analytics) to turn data-driven events into visualizations.
|
||||
|
||||
3. Create [operational panels]({{site.url}}{{site.baseurl}}/observability/operational-panels) and add visualizations to compare data the way you like.
|
||||
|
||||
4. Use [log analytics]({{site.url}}{{site.baseurl}}/observability/log-analytics) to transform unstructured log data.
|
||||
5. Use [trace analytics]({{site.url}}{{site.baseurl}}/observability/trace/index) to create traces and dive deep into your data.
|
||||
|
||||
6. Leverage [notebooks]({{site.url}}{{site.baseurl}}/observability/notebooks) to combine different visualizations and code blocks that you can share with team members.
|
||||
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
---
|
||||
layout: default
|
||||
title: Log analytics
|
||||
nav_order: 70
|
||||
---
|
||||
|
||||
# Log Ingestion
|
||||
|
||||
Log ingestion provides a way to transform unstructured log data into structured data and ingest into OpenSearch. Structured log data allows for improved queries and filtering based on the data format when searching logs for an event.
|
||||
|
||||
## Get started with log ingestion
|
||||
|
||||
OpenSearch Log Ingestion consists of three components---[Data Prepper]({{site.url}}{{site.baseurl}}/observability/data-prepper/index/), [OpenSearch]({{site.url}}{{site.baseurl}}/) and [OpenSearch Dashboards]({{site.url}}{{site.baseurl}}/)---that fit into the OpenSearch ecosystem. The Data Prepper repository has several [sample applications](https://github.com/opensearch-project/data-prepper/tree/main/examples) to help you get started.
|
||||
|
||||
### Basic flow of data
|
||||
|
||||
|
||||
|
||||
1. Log Ingestion relies on you adding log collection to your application's environment to gather and send log data.
|
||||
|
||||
(In the [example](#example) below, [FluentBit](https://docs.fluentbit.io/manual/) is used as a log collector that collects log data from a file and sends the log data to Data Prepper).
|
||||
|
||||
2. [Data Prepper]({{site.url}}{{site.baseurl}}/observability/data-prepper/index/) receives the log data, transforms the data into a structure format, and indexes it on an OpenSearch cluster.
|
||||
|
||||
3. The data can then be explored through OpenSearch search queries or the **Discover** page in OpenSearch Dashboards.
|
||||
|
||||
### Example
|
||||
|
||||
This example mimics the writing of log entries to a log file that are then processed by Data Prepper and stored in OpenSearch.
|
||||
|
||||
Download or clone the [Data Prepper repository](https://github.com/opensearch-project/data-prepper). Then navigate to `examples/log-ingestion/` and open `docker-compose.yml` in a text editor. This file contains a container for:
|
||||
|
||||
- [Fluent Bit](https://docs.fluentbit.io/manual/) (`fluent-bit`)
|
||||
- Data Prepper (`data-prepper`)
|
||||
- A single-node OpenSearch cluster (`opensearch`)
|
||||
- OpenSearch Dashboards (`opensearch-dashboards`).
|
||||
|
||||
Close the file and run `docker-compose up --build` to start the containers.
|
||||
|
||||
After the containers start, your ingestion pipeline is set up and ready to ingest log data. The `fluent-bit` container is configured to read log data from `test.log`. Run the following command to generate log data to send to the log ingestion pipeline.
|
||||
|
||||
```
|
||||
echo '63.173.168.120 - - [04/Nov/2021:15:07:25 -0500] "GET /search/tag/list HTTP/1.0" 200 5003' >> test.log
|
||||
```
|
||||
|
||||
Fluent-Bit will collect the log data and send it to Data Prepper:
|
||||
|
||||
```angular2html
|
||||
[2021/12/02 15:35:41] [ info] [output:http:http.0] data-prepper:2021, HTTP status=200
|
||||
200 OK
|
||||
```
|
||||
|
||||
Data Prepper will process the log and index it:
|
||||
|
||||
```
|
||||
2021-12-02T15:35:44,499 [log-pipeline-processor-worker-1-thread-1] INFO com.amazon.dataprepper.pipeline.ProcessWorker - log-pipeline Worker: Processing 1 records from buffer
|
||||
```
|
||||
|
||||
This should result in a single document being written to the OpenSearch cluster in the `apache-logs` index as defined in the `log_pipeline.yaml` file.
|
||||
|
||||
Run the following command to see one of the raw documents in the OpenSearch cluster:
|
||||
|
||||
```bash
|
||||
curl -X GET -u 'admin:admin' -k 'https://localhost:9200/apache_logs/_search?pretty&size=1'
|
||||
```
|
||||
|
||||
The response should show the parsed log data:
|
||||
|
||||
```
|
||||
"hits" : [
|
||||
{
|
||||
"_index" : "apache_logs",
|
||||
"_type" : "_doc",
|
||||
"_id" : "yGrJe30BgI2EWNKtDZ1g",
|
||||
"_score" : 1.0,
|
||||
"_source" : {
|
||||
"date" : 1.638459307042312E9,
|
||||
"log" : "63.173.168.120 - - [04/Nov/2021:15:07:25 -0500] \"GET /search/tag/list HTTP/1.0\" 200 5003",
|
||||
"request" : "/search/tag/list",
|
||||
"auth" : "-",
|
||||
"ident" : "-",
|
||||
"response" : "200",
|
||||
"bytes" : "5003",
|
||||
"clientip" : "63.173.168.120",
|
||||
"verb" : "GET",
|
||||
"httpversion" : "1.0",
|
||||
"timestamp" : "04/Nov/2021:15:07:25 -0500"
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
The same data can be viewed in OpenSearch Dashboards by visiting the **Discover** page and searching the `apache_logs` index. Remember, you must create the index in OpenSearch Dashboards if this is your first time searching for the index.
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
---
|
||||
layout: default
|
||||
title: Operational panels
|
||||
nav_order: 30
|
||||
---
|
||||
|
||||
# Operational panels
|
||||
|
||||
Operational panels in OpenSearch Dashboards are collections of visualizations generated using [Piped Processing Language]({{site.url}}{{site.baseurl}}/observability/ppl/index) (PPL) queries.
|
||||
|
||||
## Get started with operational panels
|
||||
|
||||
If you want to start using operational panels without adding any data, expand the **Action** menu, choose **Add samples**, and Dashboards adds a set of operational panels with saved visualizations for you to explore.
|
||||
|
||||
## Create an operational panel
|
||||
|
||||
To create an operational panel and add visualizations:
|
||||
|
||||
1. From the **Add Visualization** dropdown menu, choose **Select Existing Visualization** or **Create New Visualization**, which takes you to the [event analytics]({{site.url}}{{site.baseurl}}/observability/event-analytics) explorer, where you can use PPL to create visualizations.
|
||||
1. If you're adding already existing visualizations, choose a visualization from the dropdown menu.
|
||||
1. Choose **Add**.
|
||||
|
||||
|
||||
|
||||
To search for a particular visualization in your operation panels, use PPL queries to search for data you've already added to your panel.
|
||||
|
|
@ -47,7 +47,7 @@ search source=accounts;
|
|||
|
||||
| account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname |
|
||||
:--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :---
|
||||
| 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke
|
||||
| 1 | Amber | 880 Holmes Lane | 39225 | M | Brogan | Pyrami | IL | 32 | amberduke@pyrami.com | Duke
|
||||
| 6 | Hattie | 671 Bristol Street | 5686 | M | Dante | Netagy | TN | 36 | hattiebond@netagy.com | Bond
|
||||
| 13 | Nanette | 789 Madison Street | 32838 | F | Nogal | Quility | VA | 28 | null | Bates
|
||||
| 18 | Dale | 467 Hutchinson Court | 4180 | M | Orick | null | MD | 33 | daleadams@boink.com | Adams
|
||||
|
|
@ -57,7 +57,7 @@ search source=accounts;
|
|||
To get all documents from the `accounts` index that have either `account_number` equal to 1 or have `gender` as `F`:
|
||||
|
||||
```sql
|
||||
search source=accounts account_number=1 or gender="F";
|
||||
search source=accounts account_number=1 or gender=\"F\";
|
||||
```
|
||||
|
||||
| account_number | firstname | address | balance | gender | city | employer | state | age | email | lastname |
|
||||
|
|
@ -79,8 +79,8 @@ Field | Description | Type | Required | Default
|
|||
:--- | :--- |:--- |:--- |:---
|
||||
`int` | Retain the specified number of duplicate events for each combination. The number must be greater than 0. If you do not specify a number, only the first occurring event is kept and all other duplicates are removed from the results. | `string` | No | 1
|
||||
`keepempty` | If true, keep the document if any field in the field list has a null value or a field missing. | `nested list of objects` | No | False
|
||||
`consecutive` | If true, remove only consecutive events with duplicate combinations of values. | No | False | -
|
||||
`field-list` | Specify a comma-delimited field list. At least one field is required. | Yes | - | -
|
||||
`consecutive` | If true, remove only consecutive events with duplicate combinations of values. | `Boolean` | No | False
|
||||
`field-list` | Specify a comma-delimited field list. At least one field is required. | `String` or comma-separated list of strings | Yes | -
|
||||
|
||||
*Example 1*: Dedup by one field
|
||||
|
||||
|
|
@ -90,7 +90,7 @@ To remove duplicate documents with the same gender:
|
|||
search source=accounts | dedup gender | fields account_number, gender;
|
||||
```
|
||||
|
||||
| account_number | gender
|
||||
| account_number | gender
|
||||
:--- | :--- |
|
||||
1 | M
|
||||
13 | F
|
||||
|
|
@ -104,7 +104,7 @@ To keep two duplicate documents with the same gender:
|
|||
search source=accounts | dedup 2 gender | fields account_number, gender;
|
||||
```
|
||||
|
||||
| account_number | gender
|
||||
| account_number | gender
|
||||
:--- | :--- |
|
||||
1 | M
|
||||
6 | M
|
||||
|
|
@ -145,7 +145,7 @@ To remove duplicates of consecutive documents:
|
|||
search source=accounts | dedup gender consecutive=true | fields account_number, gender;
|
||||
```
|
||||
|
||||
| account_number | gender
|
||||
| account_number | gender
|
||||
:--- | :--- |
|
||||
1 | M
|
||||
13 | F
|
||||
|
|
@ -176,9 +176,9 @@ search source=accounts | eval doubleAge = age * 2 | fields age, doubleAge;
|
|||
|
||||
| age | doubleAge
|
||||
:--- | :--- |
|
||||
32 | 64
|
||||
36 | 72
|
||||
28 | 56
|
||||
32 | 64
|
||||
36 | 72
|
||||
28 | 56
|
||||
33 | 66
|
||||
|
||||
*Example 2*: Overwrite the existing field
|
||||
|
|
@ -191,10 +191,10 @@ search source=accounts | eval age = age + 1 | fields age;
|
|||
|
||||
| age
|
||||
:--- |
|
||||
| 33
|
||||
| 37
|
||||
| 29
|
||||
| 34
|
||||
| 33
|
||||
| 37
|
||||
| 29
|
||||
| 34
|
||||
|
||||
*Example 3*: Create a new field with a field defined with the `eval` command
|
||||
|
||||
|
|
@ -206,19 +206,19 @@ search source=accounts | eval doubleAge = age * 2, ddAge = doubleAge * 2 | field
|
|||
|
||||
| age | doubleAge | ddAge
|
||||
:--- | :--- |
|
||||
| 32 | 64 | 128
|
||||
| 36 | 72 | 144
|
||||
| 28 | 56 | 112
|
||||
| 33 | 66 | 132
|
||||
| 32 | 64 | 128
|
||||
| 36 | 72 | 144
|
||||
| 28 | 56 | 112
|
||||
| 33 | 66 | 132
|
||||
|
||||
## fields
|
||||
|
||||
Use the `field` command to keep or remove fields from a search result.
|
||||
Use the `fields` command to keep or remove fields from a search result.
|
||||
|
||||
### Syntax
|
||||
|
||||
```sql
|
||||
field [+|-] <field-list>
|
||||
fields [+|-] <field-list>
|
||||
```
|
||||
|
||||
Field | Description | Required | Default
|
||||
|
|
@ -234,11 +234,11 @@ To get `account_number`, `firstname`, and `lastname` fields from a search result
|
|||
search source=accounts | fields account_number, firstname, lastname;
|
||||
```
|
||||
|
||||
| account_number | firstname | lastname
|
||||
| account_number | firstname | lastname
|
||||
:--- | :--- |
|
||||
| 1 | Amber | Duke
|
||||
| 6 | Hattie | Bond
|
||||
| 13 | Nanette | Bates
|
||||
| 1 | Amber | Duke
|
||||
| 6 | Hattie | Bond
|
||||
| 13 | Nanette | Bates
|
||||
| 18 | Dale | Adams
|
||||
|
||||
*Example 2*: Remove specified fields from a search result
|
||||
|
|
@ -251,10 +251,10 @@ search source=accounts | fields account_number, firstname, lastname | fields - a
|
|||
|
||||
| firstname | lastname
|
||||
:--- | :--- |
|
||||
| Amber | Duke
|
||||
| Hattie | Bond
|
||||
| Nanette | Bates
|
||||
| Dale | Adams
|
||||
| Amber | Duke
|
||||
| Hattie | Bond
|
||||
| Nanette | Bates
|
||||
| Dale | Adams
|
||||
|
||||
## rename
|
||||
|
||||
|
|
@ -281,9 +281,9 @@ search source=accounts | rename account_number as an | fields an;
|
|||
|
||||
| an
|
||||
:--- |
|
||||
| 1
|
||||
| 6
|
||||
| 13
|
||||
| 1
|
||||
| 6
|
||||
| 13
|
||||
| 18
|
||||
|
||||
*Example 2*: Rename multiple fields
|
||||
|
|
@ -296,10 +296,10 @@ search source=accounts | rename account_number as an, employer as emp | fields a
|
|||
|
||||
| an | emp
|
||||
:--- | :--- |
|
||||
| 1 | Pyrami
|
||||
| 6 | Netagy
|
||||
| 1 | Pyrami
|
||||
| 6 | Netagy
|
||||
| 13 | Quility
|
||||
| 18 | null
|
||||
| 18 | null
|
||||
|
||||
## sort
|
||||
|
||||
|
|
@ -327,9 +327,9 @@ search source=accounts | sort age | fields account_number, age;
|
|||
|
||||
| account_number | age |
|
||||
:--- | :--- |
|
||||
| 13 | 28
|
||||
| 1 | 32
|
||||
| 18 | 33
|
||||
| 13 | 28
|
||||
| 1 | 32
|
||||
| 18 | 33
|
||||
| 6 | 36
|
||||
|
||||
*Example 2*: Sort by one field and return all results
|
||||
|
|
@ -342,9 +342,9 @@ search source=accounts | sort 0 age | fields account_number, age;
|
|||
|
||||
| account_number | age |
|
||||
:--- | :--- |
|
||||
| 13 | 28
|
||||
| 1 | 32
|
||||
| 18 | 33
|
||||
| 13 | 28
|
||||
| 1 | 32
|
||||
| 18 | 33
|
||||
| 6 | 36
|
||||
|
||||
*Example 3*: Sort by one field in descending order
|
||||
|
|
@ -357,9 +357,9 @@ search source=accounts | sort - age | fields account_number, age;
|
|||
|
||||
| account_number | age |
|
||||
:--- | :--- |
|
||||
| 6 | 36
|
||||
| 18 | 33
|
||||
| 1 | 32
|
||||
| 6 | 36
|
||||
| 18 | 33
|
||||
| 1 | 32
|
||||
| 13 | 28
|
||||
|
||||
*Example 4*: Specify the number of sorted documents to return
|
||||
|
|
@ -372,8 +372,8 @@ search source=accounts | sort 2 age | fields account_number, age;
|
|||
|
||||
| account_number | age |
|
||||
:--- | :--- |
|
||||
| 13 | 28
|
||||
| 1 | 32
|
||||
| 13 | 28
|
||||
| 1 | 32
|
||||
|
||||
*Example 5*: Sort by multiple fields
|
||||
|
||||
|
|
@ -385,9 +385,9 @@ search source=accounts | sort + gender, - age | fields account_number, gender, a
|
|||
|
||||
| account_number | gender | age |
|
||||
:--- | :--- | :--- |
|
||||
| 13 | F | 28
|
||||
| 6 | M | 36
|
||||
| 18 | M | 33
|
||||
| 13 | F | 28
|
||||
| 6 | M | 36
|
||||
| 18 | M | 33
|
||||
| 1 | M | 32
|
||||
|
||||
## stats
|
||||
|
|
@ -438,7 +438,7 @@ search source=accounts | stats avg(age) by gender;
|
|||
|
||||
| gender | avg(age)
|
||||
:--- | :--- |
|
||||
| F | 28.0
|
||||
| F | 28.0
|
||||
| M | 33.666666666666664
|
||||
|
||||
*Example 3*: Calculate the average and sum of a field by group
|
||||
|
|
@ -451,7 +451,7 @@ search source=accounts | stats avg(age), sum(age) by gender;
|
|||
|
||||
| gender | avg(age) | sum(age)
|
||||
:--- | :--- |
|
||||
| F | 28 | 28
|
||||
| F | 28 | 28
|
||||
| M | 33.666666666666664 | 101
|
||||
|
||||
*Example 4*: Calculate the maximum value of a field
|
||||
|
|
@ -464,7 +464,7 @@ search source=accounts | stats max(age);
|
|||
|
||||
| max(age)
|
||||
:--- |
|
||||
| 36
|
||||
| 36
|
||||
|
||||
*Example 5*: Calculate the maximum and minimum value of a field by group
|
||||
|
||||
|
|
@ -476,7 +476,7 @@ search source=accounts | stats max(age), min(age) by gender;
|
|||
|
||||
| gender | min(age) | max(age)
|
||||
:--- | :--- | :--- |
|
||||
| F | 28 | 28
|
||||
| F | 28 | 28
|
||||
| M | 32 | 36
|
||||
|
||||
## where
|
||||
|
|
@ -498,12 +498,12 @@ Field | Description | Required
|
|||
To get all documents from the `accounts` index where `account_number` is 1 or gender is `F`:
|
||||
|
||||
```sql
|
||||
search source=accounts | where account_number=1 or gender="F" | fields account_number, gender;
|
||||
search source=accounts | where account_number=1 or gender=\"F\" | fields account_number, gender;
|
||||
```
|
||||
|
||||
| account_number | gender
|
||||
:--- | :--- |
|
||||
| 1 | M
|
||||
| 1 | M
|
||||
| 13 | F
|
||||
|
||||
## head
|
||||
|
|
@ -573,7 +573,7 @@ search source=accounts | rare gender;
|
|||
|
||||
| gender
|
||||
:--- |
|
||||
| F
|
||||
| F
|
||||
| M
|
||||
|
||||
*Example 2*: Find the least common values grouped by gender
|
||||
|
|
@ -586,7 +586,7 @@ search source=accounts | rare age by gender;
|
|||
|
||||
| gender | age
|
||||
:--- | :--- |
|
||||
| F | 28
|
||||
| F | 28
|
||||
| M | 32
|
||||
| M | 33
|
||||
|
||||
|
|
@ -616,7 +616,7 @@ search source=accounts | top gender;
|
|||
|
||||
| gender
|
||||
:--- |
|
||||
| M
|
||||
| M
|
||||
| F
|
||||
|
||||
*Example 2*: Find the most common value in a field
|
||||
|
|
@ -629,7 +629,7 @@ search source=accounts | top 1 gender;
|
|||
|
||||
| gender
|
||||
:--- |
|
||||
| M
|
||||
| M
|
||||
|
||||
*Example 2*: Find the most common values grouped by gender
|
||||
|
||||
|
|
@ -641,5 +641,94 @@ search source=accounts | top 1 age by gender;
|
|||
|
||||
| gender | age
|
||||
:--- | :--- |
|
||||
| F | 28
|
||||
| F | 28
|
||||
| M | 32
|
||||
|
||||
## match
|
||||
|
||||
Use the `match` command to search documents that match a `string`, `number`, `date`, or `boolean` value for a given field.
|
||||
|
||||
### Syntax
|
||||
|
||||
```sql
|
||||
match(field_expression, query_expression[, option=<option_value>]*)
|
||||
```
|
||||
|
||||
You can specify the following options:
|
||||
|
||||
- `analyzer`
|
||||
- `auto_generate_synonyms_phrase`
|
||||
- `fuzziness`
|
||||
- `max_expansions`
|
||||
- `prefix_length`
|
||||
- `fuzzy_transpositions`
|
||||
- `fuzzy_rewrite`
|
||||
- `lenient`
|
||||
- `operator`
|
||||
- `minimum_should_match`
|
||||
- `zero_terms_query`
|
||||
- `boost`
|
||||
|
||||
*Example 1*: Search the `message` field:
|
||||
|
||||
```json
|
||||
GET my_index/_search
|
||||
{
|
||||
"query": {
|
||||
"match": {
|
||||
"message": "this is a test"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
PPL query:
|
||||
|
||||
```sql
|
||||
search source=my_index | match field=message query="this is a test"
|
||||
```
|
||||
|
||||
*Example 2*: Search the `message` field with the `operator` parameter:
|
||||
|
||||
```json
|
||||
GET my_index/_search
|
||||
{
|
||||
"query": {
|
||||
"match": {
|
||||
"message": {
|
||||
"query": "this is a test",
|
||||
"operator": "and"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
PPL query:
|
||||
|
||||
```sql
|
||||
search source=my_index | match field=message query="this is a test" operator=and
|
||||
```
|
||||
|
||||
*Example 3*: Search the `message` field with the `operator` and `zero_terms_query` parameters:
|
||||
|
||||
```json
|
||||
GET my_index/_search
|
||||
{
|
||||
"query": {
|
||||
"match": {
|
||||
"message": {
|
||||
"query": "to be or not to be",
|
||||
"operator": "and",
|
||||
"zero_terms_query": "all"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
PPL query:
|
||||
|
||||
```ppl
|
||||
search source=my_index | where match(message, "this is a test", operator=and, zero_terms_query=all)
|
||||
```
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
---
|
||||
layout: default
|
||||
title: Piped processing language
|
||||
nav_order: 42
|
||||
nav_order: 40
|
||||
has_children: true
|
||||
has_toc: false
|
||||
redirect_from:
|
||||
|
|
@ -52,9 +52,9 @@ search source=accounts
|
|||
|
||||
firstname | lastname |
|
||||
:--- | :--- |
|
||||
Amber | Duke
|
||||
Hattie | Bond
|
||||
Nanette | Bates
|
||||
Dale | Adams
|
||||
Amber | Duke
|
||||
Hattie | Bond
|
||||
Nanette | Bates
|
||||
Dale | Adams
|
||||
|
||||
|
||||
|
|
@ -9,7 +9,6 @@ nav_order: 1
|
|||
|
||||
OpenSearch Trace Analytics consists of two components---Data Prepper and the Trace Analytics OpenSearch Dashboards plugin---that fit into the OpenTelemetry and OpenSearch ecosystems. The Data Prepper repository has several [sample applications](https://github.com/opensearch-project/data-prepper/tree/main/examples) to help you get started.
|
||||
|
||||
|
||||
## Basic flow of data
|
||||
|
||||
|
||||
|
|
@ -20,10 +19,9 @@ OpenSearch Trace Analytics consists of two components---Data Prepper and the Tra
|
|||
|
||||
1. The [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/getting-started/) receives data from the application and formats it into OpenTelemetry data.
|
||||
|
||||
1. [Data Prepper]({{site.url}}{{site.baseurl}}/monitoring-plugins/trace/data-prepper/) processes the OpenTelemetry data, transforms it for use in OpenSearch, and indexes it on an OpenSearch cluster.
|
||||
|
||||
1. The [Trace Analytics OpenSearch Dashboards plugin]({{site.url}}{{site.baseurl}}/monitoring-plugins/trace/ta-dashboards/) displays the data in near real-time as a series of charts and tables, with an emphasis on service architecture, latency, error rate, and throughput.
|
||||
1. [Data Prepper]({{site.url}}{{site.baseurl}}/observability/data-prepper/index/) processes the OpenTelemetry data, transforms it for use in OpenSearch, and indexes it on an OpenSearch cluster.
|
||||
|
||||
1. The [Trace Analytics OpenSearch Dashboards plugin]({{site.url}}{{site.baseurl}}/observability/trace/ta-dashboards/) displays the data in near real-time as a series of charts and tables, with an emphasis on service architecture, latency, error rate, and throughput.
|
||||
|
||||
## Jaeger HotROD
|
||||
|
||||
|
|
@ -80,4 +78,4 @@ curl -X GET -u 'admin:admin' -k 'https://localhost:9200/otel-v1-apm-span-000001/
|
|||
|
||||
Navigate to `http://localhost:5601` in a web browser and choose **Trace Analytics**. You can see the results of your single click in the Jaeger HotROD web interface: the number of traces per API and HTTP method, latency trends, a color-coded map of the service architecture, and a list of trace IDs that you can use to drill down on individual operations.
|
||||
|
||||
If you don't see your trace, adjust the timeframe in OpenSearch Dashboards. For more information on using the plugin, see [OpenSearch Dashboards plugin]({{site.url}}{{site.baseurl}}/monitoring-plugins/trace/ta-dashboards/).
|
||||
If you don't see your trace, adjust the timeframe in OpenSearch Dashboards. For more information on using the plugin, see [OpenSearch Dashboards plugin]({{site.url}}{{site.baseurl}}/observability/trace/ta-dashboards/).
|
||||
|
|
@ -1,11 +1,9 @@
|
|||
---
|
||||
layout: default
|
||||
title: Trace analytics
|
||||
nav_order: 48
|
||||
nav_order: 60
|
||||
has_children: true
|
||||
has_toc: false
|
||||
redirect_from:
|
||||
- /monitoring-plugins/trace/
|
||||
---
|
||||
|
||||
# Trace Analytics
|
||||
|
|
@ -65,6 +65,10 @@ PUT _cluster/settings
|
|||
|
||||
You can find `opensearch.yml` in `/usr/share/opensearch/config/opensearch.yml` (Docker) or `/etc/opensearch/opensearch.yml` (most Linux distributions) on each node.
|
||||
|
||||
You can edit the `OPENSEARCH_PATH_CONF=/etc/opensearch` to change the config directory location. This variable is sourced from `/etc/default/opensearch`(Debian package) and `/etc/sysconfig/opensearch`(RPM package).
|
||||
|
||||
If you set your customized `OPENSEARCH_PATH_CONF` variable, be aware that other default environment variables will not be loaded.
|
||||
|
||||
You don't mark settings in `opensearch.yml` as persistent or transient, and settings use the flat form:
|
||||
|
||||
```yml
|
||||
|
|
|
|||
|
|
@ -228,7 +228,7 @@ POST logs-redis/_rollover
|
|||
|
||||
If you now perform a `GET` operation on the `logs-redis` data stream, you see that the generation ID is incremented from 1 to 2.
|
||||
|
||||
You can also set up an [Index State Management (ISM) policy]({{site.url}}{{site.baseurl}}/im-plugin/ism/policies/) to automate the rollover process for the data stream.
|
||||
You can also set up an [Index State Management (ISM) policy]({{site.url}}{{site.baseurl}}/im-plugin/ism/policies/) to automate the rollover process for the data stream.
|
||||
The ISM policy is applied to the backing indices at the time of their creation. When you associate a policy to a data stream, it only affects the future backing indices of that data stream.
|
||||
|
||||
You also don’t need to provide the `rollover_alias` setting, because the ISM policy infers this information from the backing index.
|
||||
|
|
@ -262,4 +262,4 @@ You can use wildcards to delete more than one data stream.
|
|||
|
||||
We recommend deleting data from a data stream using an ISM policy.
|
||||
|
||||
You can also use [asynchronous search]({{site.url}}{{site.baseurl}}/search-plugins/async/index/) and [SQL]({{site.url}}{{site.baseurl}}/search-plugins/sql/index/) and [PPL]({{site.url}}{{site.baseurl}}/search-plugins/ppl/index/) to query your data stream directly. You can also use the security plugin to define granular permissions on the data stream name.
|
||||
You can also use [asynchronous search]({{site.url}}{{site.baseurl}}/search-plugins/async/index/) and [SQL]({{site.url}}{{site.baseurl}}/search-plugins/sql/index/) and [PPL]({{site.url}}{{site.baseurl}}/observability/ppl/index/) to query your data stream directly. You can also use the security plugin to define granular permissions on the data stream name.
|
||||
|
|
|
|||
|
|
@ -195,7 +195,7 @@ You can use composable index templates to overcome these challenges. Composable
|
|||
|
||||
You can combine component templates to compose an index template.
|
||||
|
||||
Settings and mappings that you specify directly in the [create index]({{site.url}}{{site.baseurl}}/opensearch/rest-api/create-index/) request override any settings or mappings specified in an index template and its component templates.
|
||||
Settings and mappings that you specify directly in the [create index]({{site.url}}{{site.baseurl}}/opensearch/rest-api/index-apis/create-index/) request override any settings or mappings specified in an index template and its component templates.
|
||||
{: .note }
|
||||
|
||||
### Create a component template
|
||||
|
|
|
|||
|
|
@ -0,0 +1,66 @@
|
|||
---
|
||||
layout: default
|
||||
title: Ansible playbook
|
||||
parent: Install OpenSearch
|
||||
nav_order: 60
|
||||
---
|
||||
|
||||
# Ansible playbook
|
||||
|
||||
You can use an Ansible playbook to install and configure a production-ready OpenSearch cluster along with OpenSearch Dashboards.
|
||||
|
||||
The Ansible playbook only supports deployment of OpenSearch and OpenSearch Dashboards to CentOS7 hosts.
|
||||
{: .note }
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Make sure you have [Ansible](https://www.ansible.com/) and [Java 8](https://www.java.com/en/download/manual.jsp) installed.
|
||||
|
||||
## Configuration
|
||||
|
||||
1. Clone the OpenSearch [ansible-playbook](https://github.com/opensearch-project/ansible-playbook) repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/opensearch-project/ansible-playbook
|
||||
```
|
||||
|
||||
2. Configure the node properties in the `inventories/opensearch/hosts` file:
|
||||
|
||||
```bash
|
||||
ansible_host=<Public IP address> ansible_user=root ip=<Private IP address / 0.0.0.0>
|
||||
```
|
||||
|
||||
where:
|
||||
|
||||
- `ansible_host` is the IP address of the target node that you want the Ansible playbook to install OpenSearch and OpenSearch DashBoards on.
|
||||
- `ip` is the IP address that you want OpenSearch and OpenSearch DashBoards to bind to. You can specify the private IP of the target node, or localhost, or 0.0.0.0.
|
||||
|
||||
3. You can modify the default configuration values in the `inventories/opensearch/group_vars/all/all.yml` file. For example, you can increase the Java memory heap size:
|
||||
|
||||
```bash
|
||||
xms_value: 8
|
||||
xmx_value: 8
|
||||
```
|
||||
|
||||
Make sure you have direct SSH access into the root user of the target node.
|
||||
{: .note }
|
||||
|
||||
## Run OpenSearch and OpenSearch Dashboards using Ansible playbook
|
||||
|
||||
1. Run the Ansible playbook with root privileges:
|
||||
|
||||
```bash
|
||||
ansible-playbook -i inventories/opensearch/hosts opensearch.yml --extra-vars "admin_password=Test@123 kibanaserver_password=Test@6789"
|
||||
```
|
||||
|
||||
You can set the passwords for reserved users (`admin` and `kibanaserver`) using the `admin_password` and `kibanaserver_password` variables.
|
||||
|
||||
2. After the deployment process is complete, you can access OpenSearch and OpenSearch Dashboards with the username `admin` and the password that you set for the `admin_password` variable.
|
||||
|
||||
If you bind `ip` to a private IP or localhost, make sure you're logged into the server that deployed the playbook to access OpenSearch and OpenSearch Dashboards:
|
||||
|
||||
```bash
|
||||
curl https://localhost:9200 -u 'admin:Test@123' --insecure
|
||||
```
|
||||
|
||||
If you bind `ip` to 0.0.0.0, then replace `localhost` with the public IP or the private IP (if it's in the same network).
|
||||
|
|
@ -87,7 +87,7 @@ services:
|
|||
networks:
|
||||
- opensearch-net
|
||||
opensearch-dashboards
|
||||
image: opensearchproject/opensearch-dashboards:{{site.opensearch_version}}
|
||||
image: opensearchproject/opensearch-dashboards:{{site.opensearch_dashboards_version}}
|
||||
container_name: opensearch-dashboards
|
||||
ports:
|
||||
- 5601:5601
|
||||
|
|
@ -129,8 +129,8 @@ plugins.security.enable_snapshot_restore_privilege: true
|
|||
plugins.security.check_snapshot_restore_write_privileges: true
|
||||
plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
|
||||
cluster.routing.allocation.disk.threshold_enabled: false
|
||||
plugins.security.audit.config.disabled_rest_categories: NONE
|
||||
plugins.security.audit.config.disabled_transport_categories: NONE
|
||||
opendistro_security.audit.config.disabled_rest_categories: NONE
|
||||
opendistro_security.audit.config.disabled_transport_categories: NONE
|
||||
```
|
||||
|
||||
Use this same override process to specify new [authentication settings]({{site.url}}{{site.baseurl}}/security-plugin/configuration/configuration/) in `/usr/share/opensearch/plugins/opensearch-security/securityconfig/config.yml`, as well as new default [internal users, roles, mappings, action groups, and tenants]({{site.url}}{{site.baseurl}}/security-plugin/configuration/yaml/).
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ nav_order: 3
|
|||
You can pull the OpenSearch Docker image just like any other image:
|
||||
|
||||
```bash
|
||||
docker pull opensearchproject/opensearch:{{site.opensearch_version}}
|
||||
docker pull opensearchproject/opensearch-dashboards:{{site.opensearch_version}}
|
||||
docker pull opensearchproject/opensearch:latest
|
||||
docker pull opensearchproject/opensearch-dashboards:latest
|
||||
```
|
||||
|
||||
To check available versions, see [Docker Hub](https://hub.docker.com/u/opensearchproject).
|
||||
|
|
@ -77,6 +77,8 @@ To stop the cluster and delete all data volumes, run:
|
|||
docker-compose down -v
|
||||
```
|
||||
|
||||
If you're running your cluster in a production environment, be sure to refer to [Important settings]({{site.url}}{{site.baseurl}}/opensearch/install/important-settings) when configuring your machine and cluster.
|
||||
{: .note}
|
||||
|
||||
#### Sample Docker Compose file
|
||||
|
||||
|
|
@ -131,7 +133,7 @@ services:
|
|||
networks:
|
||||
- opensearch-net
|
||||
opensearch-dashboards:
|
||||
image: opensearchproject/opensearch-dashboards:{{site.opensearch_version}}
|
||||
image: opensearchproject/opensearch-dashboards:{{site.opensearch_dashboards_version}}
|
||||
container_name: opensearch-dashboards
|
||||
ports:
|
||||
- 5601:5601
|
||||
|
|
@ -322,3 +324,63 @@ In this case, `opensearch.yml` is a "vanilla" version of the file with no plugin
|
|||
cluster.name: "docker-cluster"
|
||||
network.host: 0.0.0.0
|
||||
```
|
||||
|
||||
## Sample Docker Compose file for development
|
||||
|
||||
You can use this sample file as a development environment.
|
||||
|
||||
This sample file starts one OpenSearch node and a container for OpenSearch Dashboards with the security plugin disabled.
|
||||
|
||||
```yml
|
||||
version: '3'
|
||||
services:
|
||||
opensearch-node1:
|
||||
image: opensearchproject/opensearch:{{site.opensearch_version}}
|
||||
container_name: opensearch-node1
|
||||
environment:
|
||||
- cluster.name=opensearch-cluster
|
||||
- node.name=opensearch-node1
|
||||
- bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
|
||||
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
|
||||
- "DISABLE_INSTALL_DEMO_CONFIG=true" # disables execution of install_demo_configuration.sh bundled with security plugin, which installs demo certificates and security configurations to OpenSearch
|
||||
- "DISABLE_SECURITY_PLUGIN=true" # disables security plugin entirely in OpenSearch by setting plugins.security.disabled: true in opensearch.yml
|
||||
- "discovery.type=single-node" # disables bootstrap checks that are enabled when network.host is set to a non-loopback address
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
nofile:
|
||||
soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
|
||||
hard: 65536
|
||||
volumes:
|
||||
- opensearch-data1:/usr/share/opensearch/data
|
||||
ports:
|
||||
- 9200:9200
|
||||
- 9600:9600 # required for Performance Analyzer
|
||||
networks:
|
||||
- opensearch-net
|
||||
|
||||
opensearch-dashboards:
|
||||
image: opensearchproject/opensearch-dashboards:{{site.opensearch_dashboards_version}}
|
||||
container_name: opensearch-dashboards
|
||||
ports:
|
||||
- 5601:5601
|
||||
expose:
|
||||
- "5601"
|
||||
environment:
|
||||
- 'OPENSEARCH_HOSTS=["http://opensearch-node1:9200"]'
|
||||
- "DISABLE_SECURITY_DASHBOARDS_PLUGIN=true" # disables security dashboards plugin in OpenSearch Dashboards
|
||||
networks:
|
||||
- opensearch-net
|
||||
|
||||
volumes:
|
||||
opensearch-data1:
|
||||
|
||||
networks:
|
||||
opensearch-net:
|
||||
```
|
||||
|
||||
The environment variable `"DISABLE_SECURITY_DASHBOARDS_PLUGIN=true"` disables the security dashboards plugin in OpenSearch Dashboards by removing the security dashboards plugin folder, removing all related settings in the `opensearch_dashboards.yml` file, and setting the `opensearch.hosts` entry protocol from HTTPS to HTTP.
|
||||
You can't reverse this step as the security dashboards plugin is removed in the process.
|
||||
To re-enable security for OpenSearch Dashboards, start a new container and set `DISABLE_SECURITY_DASHBOARDS_PLUGIN` to false or leave it unset.
|
||||
{: .note}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ Resource | Description
|
|||
The specification in the default Helm chart supports many standard use cases and setups. You can modify the default chart to configure your desired specifications and set Transport Layer Security (TLS) and role-based access control (RBAC).
|
||||
|
||||
For information about the default configuration, steps to configure security, and configurable parameters, see the
|
||||
[README](https://github.com/opensearch-project/opensearch-devops/blob/main/Helm/README.md).
|
||||
[README](https://github.com/opensearch-project/helm-charts/blob/main/README.md).
|
||||
|
||||
The instructions here assume you have a Kubernetes cluster with Helm preinstalled. See the [Kubernetes documentation](https://kubernetes.io/docs/setup/) for steps to configure a Kubernetes cluster and the [Helm documentation](https://helm.sh/docs/intro/install/) to install Helm.
|
||||
{: .note }
|
||||
|
|
@ -31,12 +31,38 @@ The default Helm chart deploys a three-node cluster. We recommend that you have
|
|||
|
||||
## Install OpenSearch using Helm
|
||||
|
||||
1. Clone the [helm-charts](https://github.com/opensearch-project/helm-charts) repository:
|
||||
1. Add `opensearch` [helm-charts](https://github.com/opensearch-project/helm-charts) repository to Helm:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/opensearch-project/helm-charts
|
||||
helm repo add opensearch https://opensearch-project.github.io/helm-charts/
|
||||
```
|
||||
|
||||
1. Update the available charts locally from charts repositories:
|
||||
|
||||
```bash
|
||||
helm repo update
|
||||
```
|
||||
|
||||
1. To search for the OpenSearch-related Helm charts:
|
||||
|
||||
```bash
|
||||
helm search repo opensearch
|
||||
```
|
||||
|
||||
```bash
|
||||
NAME CHART VERSION APP VERSION DESCRIPTION
|
||||
opensearch/opensearch 1.0.7 1.0.0 A Helm chart for OpenSearch
|
||||
opensearch/opensearch-dashboards 1.0.4 1.0.0 A Helm chart for OpenSearch Dashboards
|
||||
```
|
||||
|
||||
1. Deploy OpenSearch:
|
||||
|
||||
```bash
|
||||
helm install my-deployment opensearch/opensearch
|
||||
```
|
||||
|
||||
You can also build the `opensearch-1.0.0.tgz` file manually:
|
||||
|
||||
1. Change to the `opensearch` directory:
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -5,9 +5,9 @@ parent: Install OpenSearch
|
|||
nav_order: 90
|
||||
---
|
||||
|
||||
# Standalone OpenSearch plugin installation
|
||||
# OpenSearch plugin installation
|
||||
|
||||
If you don't want to use the all-in-one OpenSearch installation options, you can install the individual plugins on a compatible OpenSearch cluster, just like any other plugin.
|
||||
You can install individual plugins on an OpenSearch cluster.
|
||||
|
||||
|
||||
---
|
||||
|
|
@ -19,7 +19,29 @@ If you don't want to use the all-in-one OpenSearch installation options, you can
|
|||
|
||||
---
|
||||
|
||||
## Plugin compatibility
|
||||
## Install a plugin
|
||||
|
||||
```
|
||||
bin/opensearch-plugin install <plugin-name>
|
||||
```
|
||||
Restart your OpenSearch node after installing a plugin.
|
||||
|
||||
## Remove a plugin
|
||||
|
||||
```
|
||||
bin/opensearch-plugin remove <plugin-name>
|
||||
```
|
||||
Restart your OpenSearch node after removing a plugin.
|
||||
|
||||
## List installed plugins
|
||||
|
||||
To check which plugins are currently installed:
|
||||
|
||||
```
|
||||
bin/opensearch-plugin list
|
||||
```
|
||||
|
||||
## Available plugins
|
||||
|
||||
<table>
|
||||
<thead style="text-align: left">
|
||||
|
|
@ -29,6 +51,62 @@ If you don't want to use the all-in-one OpenSearch installation options, you can
|
|||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>1.2.0</td>
|
||||
<td>
|
||||
<pre>analysis-icu 1.1.0
|
||||
analysis-kuromoji 1.1.0
|
||||
analysis-nori 1.1.0
|
||||
analysis-phonetic 1.1.0
|
||||
analysis-smartcn 1.1.0
|
||||
analysis-stempel 1.1.0
|
||||
analysis-ukrainian 1.1.0
|
||||
discovery-azure-classic 1.1.0
|
||||
discovery-ec2 1.1.0
|
||||
discovery-gce 1.1.0
|
||||
ingest-attachment 1.1.0
|
||||
mapper-annotated-text 1.1.0
|
||||
mapper-murmur3 1.1.0
|
||||
mapper-size 1.1.0
|
||||
opensearch-alerting 1.2.0.0
|
||||
opensearch-anomaly-detection 1.2.0.0
|
||||
opensearch-asynchronous-search 1.2.0.0
|
||||
opensearch-cross-cluster-replication 1.2.0.0
|
||||
opensearch-index-management 1.2.0.0
|
||||
opensearch-job-scheduler 1.2.0.0
|
||||
opensearch-knn 1.2.0.0
|
||||
opensearch-notebooks 1.2.0.0
|
||||
opensearch-performance-analyzer 1.2.0.0
|
||||
opensearch-reports-scheduler 1.2.0.0
|
||||
opensearch-security 1.2.0.0
|
||||
opensearch-sql 1.2.0.0
|
||||
repository-azure 1.1.0
|
||||
repository-gcs 1.1.0
|
||||
repository-hdfs 1.1.0
|
||||
repository-s3 1.1.0
|
||||
store-smb 1.1.0
|
||||
transport-nio 1.1.0
|
||||
</pre>
|
||||
<tr>
|
||||
<td>1.1.0</td>
|
||||
<td>
|
||||
<pre>opensearch-alerting 1.1.0.0
|
||||
opensearch-anomaly-detection 1.1.0.0
|
||||
opensearch-asynchronous-search 1.1.0.0
|
||||
opensearch-cross-cluster-replication 1.1.0.0
|
||||
opensearch-index-management 1.1.0.0
|
||||
opensearch-job-scheduler 1.1.0.0
|
||||
opensearch-knn 1.1.0.0
|
||||
opensearch-notebooks 1.1.0.0
|
||||
opensearch-performance-analyzer 1.1.0.0
|
||||
opensearch-reports-scheduler 1.1.0.0
|
||||
opensearch-security 1.1.0.0
|
||||
opensearch-sql 1.1.0.0
|
||||
</pre>
|
||||
</td>
|
||||
</tr>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>1.0.1</td>
|
||||
<td>
|
||||
|
|
@ -65,217 +143,3 @@ opensearch-sql 1.0.0.0
|
|||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
To install plugins manually, you must have the exact version of OpenSearch installed, down to the minor version.
|
||||
|
||||
{% comment %}
|
||||
|
||||
To get a list of available OpenSearch versions on CentOS 7 and Amazon Linux 2, run the following command:
|
||||
|
||||
```bash
|
||||
sudo yum list opensearch-oss --showduplicates
|
||||
```
|
||||
|
||||
Then you can specify the version that you need:
|
||||
|
||||
```bash
|
||||
sudo yum install opensearch-oss-6.7.1
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
## Install plugins
|
||||
|
||||
Navigate to the OpenSearch home directory (most likely, it is `/usr/share/opensearch`), and run the install command for each plugin.
|
||||
|
||||
|
||||
### Security
|
||||
|
||||
```bash
|
||||
sudo bin/opensearch-plugin install https://d3g5vo6xdbdb9a.cloudfront.net/downloads/opensearch-plugins/opensearch-security/opensearch-security-{{site.opensearch_major_minor_version}}.1.0.zip
|
||||
```
|
||||
|
||||
After installing the security plugin, you can run `sudo sh /usr/share/opensearch/plugins/opensearch-security/tools/install_demo_configuration.sh` to quickly get started with demo certificates. Otherwise, you must configure it manually and run [securityadmin.sh]({{site.url}}{{site.baseurl}}/security-plugin/configuration/security-admin/).
|
||||
|
||||
The security plugin has a corresponding [OpenSearch Dashboards plugin]({{site.url}}{{site.baseurl}}/opensearch-dashboards/install/plugins) that you probably want to install as well.
|
||||
|
||||
|
||||
### Job scheduler
|
||||
|
||||
```bash
|
||||
sudo bin/opensearch-plugin install https://d3g5vo6xdbdb9a.cloudfront.net/downloads/opensearch-plugins/opensearch-job-scheduler/opensearch-job-scheduler-{{site.opensearch_major_minor_version}}.0.0.zip
|
||||
```
|
||||
|
||||
|
||||
### Alerting
|
||||
|
||||
```bash
|
||||
sudo bin/opensearch-plugin install https://d3g5vo6xdbdb9a.cloudfront.net/downloads/opensearch-plugins/opensearch-alerting/opensearch-alerting-{{site.opensearch_major_minor_version}}.1.0.zip
|
||||
```
|
||||
|
||||
To install Alerting, you must first install the Job Scheduler plugin. Alerting has a corresponding [OpenSearch Dashboards plugin]({{site.url}}{{site.baseurl}}/opensearch-dashboards/install/plugins/) that you probably want to install as well.
|
||||
|
||||
|
||||
### SQL
|
||||
|
||||
```bash
|
||||
sudo bin/opensearch-plugin install https://d3g5vo6xdbdb9a.cloudfront.net/downloads/opensearch-plugins/opensearch-sql/opensearch-sql-{{site.opensearch_major_minor_version}}.2.0.zip
|
||||
```
|
||||
|
||||
|
||||
### Reports scheduler
|
||||
|
||||
```bash
|
||||
sudo bin/opensearch-plugin install https://d3g5vo6xdbdb9a.cloudfront.net/downloads/opensearch-plugins/opensearch-reports-scheduler/opensearch-reports-scheduler-{{site.opensearch_major_minor_version}}.0.0.zip
|
||||
```
|
||||
|
||||
|
||||
### Index State Management
|
||||
|
||||
```bash
|
||||
sudo bin/opensearch-plugin install https://d3g5vo6xdbdb9a.cloudfront.net/downloads/opensearch-plugins/opensearch-index-management/opensearch-index-management-{{site.opensearch_major_minor_version}}.2.0.zip
|
||||
```
|
||||
|
||||
To install Index State Management, you must first install the Job Scheduler plugin. ISM has a corresponding [OpenSearch Dashboards plugin]({{site.url}}{{site.baseurl}}/opensearch-dashboards/install/plugins/) that you probably want to install as well.
|
||||
|
||||
|
||||
### k-NN
|
||||
|
||||
k-NN is only available as part of the all-in-one installs: Docker, RPM, and Debian.
|
||||
|
||||
|
||||
### Anomaly detection
|
||||
|
||||
```bash
|
||||
sudo bin/opensearch-plugin install https://d3g5vo6xdbdb9a.cloudfront.net/downloads/opensearch-plugins/opensearch-anomaly-detection/opensearch-anomaly-detection-{{site.opensearch_major_minor_version}}.0.0.zip
|
||||
```
|
||||
|
||||
|
||||
### Asynchronous search
|
||||
|
||||
```bash
|
||||
sudo bin/opensearch-plugin install https://d3g5vo6xdbdb9a.cloudfront.net/downloads/opensearch-plugins/opensearch-asynchronous-search/opensearch-asynchronous-search-{{site.opensearch_major_minor_version}}.0.1.zip
|
||||
```
|
||||
|
||||
|
||||
### Performance Analyzer
|
||||
|
||||
```bash
|
||||
sudo bin/opensearch-plugin install https://d3g5vo6xdbdb9a.cloudfront.net/downloads/opensearch-plugins/performance-analyzer/opensearch-performance-analyzer-{{site.opensearch_major_minor_version}}.0.0.zip
|
||||
```
|
||||
|
||||
Performance Analyzer requires some manual configuration after installing the plugin:
|
||||
|
||||
1. Create `/usr/lib/systemd/system/opensearch-performance-analyzer.service` based on [this file](https://github.com/opensearch-project/performance-analyzer/blob/master/packaging/opensearch-performance-analyzer.service).
|
||||
|
||||
1. Make the CLI executable:
|
||||
|
||||
```bash
|
||||
sudo chmod +x /usr/share/opensearch/bin/performance-analyzer-agent-cli
|
||||
```
|
||||
|
||||
1. Run the appropriate `postinst` script for your Linux distribution:
|
||||
|
||||
```bash
|
||||
# Debian-based distros
|
||||
sudo sh /usr/share/opensearch/plugins/opensearch-performance-analyzer/install/deb/postinst.sh 1
|
||||
|
||||
# RPM distros
|
||||
sudo sh /usr/share/opensearch/plugins/opensearch-performance-analyzer/install/rpm/postinst.sh 1
|
||||
```
|
||||
|
||||
1. Make Performance Analyzer accessible outside of the host machine
|
||||
|
||||
```bash
|
||||
cd /usr/share/opensearch # navigate to the OpenSearch home directory
|
||||
cd plugins/opensearch_performance_analyzer/pa_config/
|
||||
vi performance-analyzer.properties
|
||||
```
|
||||
|
||||
Uncomment the line `#webservice-bind-host` and set it to `0.0.0.0`:
|
||||
|
||||
```bash
|
||||
# ======================== OpenSearch performance analyzer plugin config =========================
|
||||
|
||||
# NOTE: this is an example for Linux. Please modify the config accordingly if you are using it under other OS.
|
||||
|
||||
# WebService bind host; default to all interfaces
|
||||
webservice-bind-host = 0.0.0.0
|
||||
|
||||
# Metrics data location
|
||||
metrics-location = /dev/shm/performanceanalyzer/
|
||||
|
||||
# Metrics deletion interval (minutes) for metrics data.
|
||||
# Interval should be between 1 to 60.
|
||||
metrics-deletion-interval = 1
|
||||
|
||||
# If set to true, the system cleans up the files behind it. So at any point, we should expect only 2
|
||||
# metrics-db-file-prefix-path files. If set to false, no files are cleaned up. This can be useful, if you are archiving
|
||||
# the files and wouldn't like for them to be cleaned up.
|
||||
cleanup-metrics-db-files = true
|
||||
|
||||
# WebService exposed by App's port
|
||||
webservice-listener-port = 9600
|
||||
|
||||
# Metric DB File Prefix Path location
|
||||
metrics-db-file-prefix-path = /tmp/metricsdb_
|
||||
|
||||
https-enabled = false
|
||||
|
||||
#Setup the correct path for certificates
|
||||
certificate-file-path = specify_path
|
||||
|
||||
private-key-file-path = specify_path
|
||||
|
||||
# Plugin Stats Metadata file name, expected to be in the same location
|
||||
plugin-stats-metadata = plugin-stats-metadata
|
||||
|
||||
# Agent Stats Metadata file name, expected to be in the same location
|
||||
agent-stats-metadata = agent-stats-metadata
|
||||
```
|
||||
|
||||
1. Start the OpenSearch service:
|
||||
|
||||
```bash
|
||||
sudo systemctl start opensearch.service
|
||||
```
|
||||
|
||||
1. Send a test request:
|
||||
|
||||
```bash
|
||||
curl -XGET "localhost:9600/_plugins/_performanceanalyzer/metrics?metrics=Latency,CPU_Utilization&agg=avg,max&dim=ShardID&nodes=all"
|
||||
```
|
||||
{% endcomment %}
|
||||
|
||||
## List installed plugins
|
||||
|
||||
To check your installed plugins:
|
||||
|
||||
```bash
|
||||
sudo bin/opensearch-plugin list
|
||||
```
|
||||
|
||||
|
||||
## Remove plugins
|
||||
|
||||
If you are removing Performance Analyzer, see below. Otherwise, you can remove the plugin with a single command:
|
||||
|
||||
```bash
|
||||
sudo bin/opensearch-plugin remove <plugin-name>
|
||||
```
|
||||
|
||||
Then restart OpenSearch on the node:
|
||||
|
||||
```bash
|
||||
sudo systemctl restart opensearch.service
|
||||
```
|
||||
|
||||
## Update plugins
|
||||
|
||||
OpenSearch doesn't update plugins. Instead, you have to remove and reinstall them:
|
||||
|
||||
```bash
|
||||
sudo bin/opensearch-plugin remove <plugin-name>
|
||||
sudo bin/opensearch-plugin install <plugin-name>
|
||||
```
|
||||
|
|
|
|||
|
|
@ -18,17 +18,20 @@ The tarball supports most Linux distributions, including CentOS 7, Amazon Linux
|
|||
```bash
|
||||
# x64
|
||||
tar -zxf opensearch-{{site.opensearch_version}}-linux-x64.tar.gz
|
||||
cd opensearch-{{site.opensearch_version}}{% comment %}# ARM64
|
||||
cd opensearch-{{site.opensearch_version}}
|
||||
# ARM64
|
||||
tar -zxf opensearch-{{site.opensearch_version}}-linux-arm64.tar.gz
|
||||
cd opensearch-{{site.opensearch_version}}{% endcomment %}
|
||||
cd opensearch-{{site.opensearch_version}}
|
||||
```
|
||||
|
||||
1. Run OpenSearch:
|
||||
1. Run OpenSearch with the demo security configuration:
|
||||
|
||||
```bash
|
||||
./opensearch-tar-install.sh
|
||||
```
|
||||
|
||||
If you configure the security plugin for production use (or disable it), you can run OpenSearch using `./bin/opensearch`.
|
||||
|
||||
1. Open a second terminal session, and send requests to the server to verify that OpenSearch is up and running:
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -21,11 +21,9 @@ GET _cluster/settings?include_defaults=true
|
|||
```json
|
||||
PUT _cluster/settings
|
||||
{
|
||||
"persistent": {
|
||||
"action": {
|
||||
"auto_create_index": false
|
||||
}
|
||||
}
|
||||
"persistent":{
|
||||
"action.auto_create_index": false
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -59,11 +57,9 @@ For a PUT operation, the request body must contain `transient` or `persistent`,
|
|||
```json
|
||||
PUT _cluster/settings
|
||||
{
|
||||
"persistent": {
|
||||
"cluster": {
|
||||
"max_shards_per_node": 500
|
||||
}
|
||||
}
|
||||
"persistent":{
|
||||
"cluster.max_shards_per_node": 500
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -74,12 +70,12 @@ For more information about transient settings, persistent settings, and preceden
|
|||
|
||||
```json
|
||||
{
|
||||
"acknowledged": true,
|
||||
"persistent": {
|
||||
"cluster": {
|
||||
"max_shards_per_node": "500"
|
||||
}
|
||||
},
|
||||
"transient": {}
|
||||
"acknowledged":true,
|
||||
"persistent":{
|
||||
"cluster":{
|
||||
"max_shards_per_node":"500"
|
||||
}
|
||||
},
|
||||
"transient":{}
|
||||
}
|
||||
```
|
||||
|
|
|
|||
|
|
@ -0,0 +1,30 @@
|
|||
---
|
||||
layout: default
|
||||
title: Shard indexing backpressure
|
||||
nav_order: 62
|
||||
has_children: true
|
||||
---
|
||||
|
||||
# Shard indexing backpressure
|
||||
|
||||
Shard indexing backpressure is a smart rejection mechanism at a per-shard level that dynamically rejects indexing requests when your cluster is under strain. It propagates a backpressure that transfers requests from an overwhelmed node or shard to other nodes or shards that are still healthy.
|
||||
|
||||
With shard indexing backpressure, you can prevent nodes in your cluster from running into cascading failures due to performance degradation caused by slow nodes, stuck tasks, resource-intensive requests, traffic surges, skewed shard allocations, and so on.
|
||||
|
||||
Shard indexing backpressure comes into effect only when one primary and one secondary parameter is breached.
|
||||
|
||||
## Primary parameters
|
||||
|
||||
Primary parameters are early indicators that a cluster is under strain:
|
||||
|
||||
- Shard memory limit breach: If the memory usage of a shard exceeds 95% of its allocated memory, this limit is breached.
|
||||
- Node memory limit breach: If the memory usage of a node exceeds 70% of its allocated memory, this limit is breached.
|
||||
|
||||
The breach of primary parameters doesn’t cause any actual request rejections, it just triggers an evaluation of the secondary parameters.
|
||||
|
||||
## Secondary parameters
|
||||
|
||||
Secondary parameters check the performance at the shard level to confirm that the cluster is under strain:
|
||||
|
||||
- Throughput: If the throughput at the shard level decreases significantly in its historic view, this limit is breached.
|
||||
- Successful Request: If the number of pending requests increases significantly in its historic view, this limit is breached.
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
---
|
||||
layout: default
|
||||
title: Settings
|
||||
parent: Shard indexing backpressure
|
||||
nav_order: 1
|
||||
has_children: false
|
||||
---
|
||||
|
||||
# Settings
|
||||
|
||||
Shard indexing backpressure adds several settings to the standard OpenSearch cluster settings. They are dynamic, so you can change the default behavior of this feature without restarting your cluster.
|
||||
|
||||
## High-level controls
|
||||
|
||||
The high-level controls allow you to turn the shard indexing backpressure feature on or off.
|
||||
|
||||
Setting | Default | Description
|
||||
:--- | :--- | :---
|
||||
`shard_indexing_pressure.enable` | False | Change to `true` to enable shard indexing backpressure.
|
||||
`shard_indexing_pressure.enforced` | False | Run shard indexing backpressure in shadow mode or enforced mode. In shadow mode (value set as `false`), shard indexing backpressure tracks all granular-level metrics, but it doesn't actually reject any indexing requests. In enforced mode (value set as `true`), shard indexing backpressure rejects any requests to the cluster that might cause a dip in its performance.
|
||||
|
||||
## Node-level limits
|
||||
|
||||
Node-level limits allow you to control memory usage on a node.
|
||||
|
||||
Setting | Default | Description
|
||||
:--- | :--- | :---
|
||||
`shard_indexing_pressure.primary_parameter.node.soft_limit` | 70% | Define the percentage of the node-level memory threshold that acts as a soft indicator for strain on a node.
|
||||
|
||||
## Shard-level limits
|
||||
|
||||
Shard-level limits allow you to control memory usage on a shard.
|
||||
|
||||
Setting | Default | Description
|
||||
:--- | :--- | :---
|
||||
`shard_indexing_pressure.primary_parameter.shard.min_limit` | 0.001d | Specify the minimum assigned quota for a new shard in any role (coordinator, primary, or replica). Shard indexing backpressure increases or decreases this allocated quota based on the inflow of traffic for the shard.
|
||||
`shard_indexing_pressure.operating_factor.lower` | 75% | Specify the lower occupancy limit of the allocated quota of memory for the shard. If the total memory usage of a shard is below this limit, shard indexing backpressure decreases the current allocated memory for that shard.
|
||||
`shard_indexing_pressure.operating_factor.optimal` | 85% | Specify the optimal occupancy of the allocated quota of memory for the shard. If the total memory usage of a shard is at this level, shard indexing backpressure doesn't change the current allocated memory for that shard.
|
||||
`shard_indexing_pressure.operating_factor.upper` | 95% | Specify the upper occupancy limit of the allocated quota of memory for the shard. If the total memory usage of a shard is above this limit, shard indexing backpressure increases the current allocated memory for that shard.
|
||||
|
||||
## Performance degradation factors
|
||||
|
||||
The performance degradation factors allow you to control the dynamic performance thresholds for a shard.
|
||||
|
||||
Setting | Default | Description
|
||||
:--- | :--- | :---
|
||||
`shard_indexing_pressure.secondary_parameter.throughput.request_size_window` | 2,000 | The number of requests in the sampling window size on a shard. Shard indexing backpressure compares the overall performance of requests with the requests in the sample window to detect any performance degradation.
|
||||
`shard_indexing_pressure.secondary_parameter.throughput.degradation_factor` | 5x | The degradation factor per unit byte for a request. This parameter determines the threshold for any latency spikes. The default value is 5x, which implies that if the latency shoots up 5 times in the historic view, shard indexing backpressure marks it as a performance degradation.
|
||||
`shard_indexing_pressure.secondary_parameter.successful_request.elapsed_timeout` | 300000 ms | The amount of time a request is pending in a cluster. This parameter helps identify any stuck-request scenarios.
|
||||
`shard_indexing_pressure.secondary_parameter.successful_request.max_outstanding_requests` | 100 | The maximum number of pending requests in a cluster.
|
||||
|
|
@ -0,0 +1,493 @@
|
|||
---
|
||||
layout: default
|
||||
title: Stats API
|
||||
parent: Shard indexing backpressure
|
||||
nav_order: 2
|
||||
has_children: false
|
||||
---
|
||||
|
||||
# Stats API
|
||||
|
||||
Use the stats operation to monitor shard indexing backpressure.
|
||||
|
||||
## Stats
|
||||
Introduced 1.2
|
||||
{: .label .label-purple }
|
||||
|
||||
Returns node-level and shard-level stats for indexing request rejections.
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
GET _nodes/_local/stats/shard_indexing_pressure
|
||||
```
|
||||
|
||||
If `enforced` is `true`:
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"_nodes": {
|
||||
"total": 1,
|
||||
"successful": 1,
|
||||
"failed": 0
|
||||
},
|
||||
"cluster_name": "runTask",
|
||||
"nodes": {
|
||||
"q3e1dQjFSqyPSLAgpyQlfw": {
|
||||
"timestamp": 1613072111162,
|
||||
"name": "runTask-0",
|
||||
"transport_address": "127.0.0.1:9300",
|
||||
"host": "127.0.0.1",
|
||||
"ip": "127.0.0.1:9300",
|
||||
"roles": [
|
||||
"data",
|
||||
"ingest",
|
||||
"master",
|
||||
"remote_cluster_client"
|
||||
],
|
||||
"attributes": {
|
||||
"testattr": "test"
|
||||
},
|
||||
"shard_indexing_pressure": {
|
||||
"stats": {
|
||||
"[index_name][0]": {
|
||||
"memory": {
|
||||
"current": {
|
||||
"coordinating_in_bytes": 0,
|
||||
"primary_in_bytes": 0,
|
||||
"replica_in_bytes": 0
|
||||
},
|
||||
"total": {
|
||||
"coordinating_in_bytes": 299,
|
||||
"primary_in_bytes": 299,
|
||||
"replica_in_bytes": 0
|
||||
}
|
||||
},
|
||||
"rejection": {
|
||||
"coordinating": {
|
||||
"coordinating_rejections": 0,
|
||||
"breakup": {
|
||||
"node_limits": 0,
|
||||
"no_successful_request_limits": 0,
|
||||
"throughput_degradation_limits": 0
|
||||
}
|
||||
},
|
||||
"primary": {
|
||||
"primary_rejections": 0,
|
||||
"breakup": {
|
||||
"node_limits": 0,
|
||||
"no_successful_request_limits": 0,
|
||||
"throughput_degradation_limits": 0
|
||||
}
|
||||
},
|
||||
"replica": {
|
||||
"replica_rejections": 0,
|
||||
"breakup": {
|
||||
"node_limits": 0,
|
||||
"no_successful_request_limits": 0,
|
||||
"throughput_degradation_limits": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"last_successful_timestamp": {
|
||||
"coordinating_last_successful_request_timestamp_in_millis": 1613072107990,
|
||||
"primary_last_successful_request_timestamp_in_millis": 0,
|
||||
"replica_last_successful_request_timestamp_in_millis": 0
|
||||
},
|
||||
"indexing": {
|
||||
"coordinating_time_in_millis": 96,
|
||||
"coordinating_count": 1,
|
||||
"primary_time_in_millis": 0,
|
||||
"primary_count": 0,
|
||||
"replica_time_in_millis": 0,
|
||||
"replica_count": 0
|
||||
},
|
||||
"memory_allocation": {
|
||||
"current": {
|
||||
"current_coordinating_and_primary_bytes": 0,
|
||||
"current_replica_bytes": 0
|
||||
},
|
||||
"limit": {
|
||||
"current_coordinating_and_primary_limits_in_bytes": 51897,
|
||||
"current_replica_limits_in_bytes": 77845
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"total_rejections_breakup": {
|
||||
"node_limits": 0,
|
||||
"no_successful_request_limits": 0,
|
||||
"throughput_degradation_limits": 0
|
||||
},
|
||||
"enabled": true,
|
||||
"enforced" : true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
If `enforced` is `false`:
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"_nodes": {
|
||||
"total": 1,
|
||||
"successful": 1,
|
||||
"failed": 0
|
||||
},
|
||||
"cluster_name": "runTask",
|
||||
"nodes": {
|
||||
"q3e1dQjFSqyPSLAgpyQlfw": {
|
||||
"timestamp": 1613072111162,
|
||||
"name": "runTask-0",
|
||||
"transport_address": "127.0.0.1:9300",
|
||||
"host": "127.0.0.1",
|
||||
"ip": "127.0.0.1:9300",
|
||||
"roles": [
|
||||
"data",
|
||||
"ingest",
|
||||
"master",
|
||||
"remote_cluster_client"
|
||||
],
|
||||
"attributes": {
|
||||
"testattr": "test"
|
||||
},
|
||||
"shard_indexing_pressure": {
|
||||
"stats": {
|
||||
"[index_name][0]": {
|
||||
"memory": {
|
||||
"current": {
|
||||
"coordinating_in_bytes": 0,
|
||||
"primary_in_bytes": 0,
|
||||
"replica_in_bytes": 0
|
||||
},
|
||||
"total": {
|
||||
"coordinating_in_bytes": 299,
|
||||
"primary_in_bytes": 299,
|
||||
"replica_in_bytes": 0
|
||||
}
|
||||
},
|
||||
"rejection": {
|
||||
"coordinating": {
|
||||
"coordinating_rejections": 0,
|
||||
"breakup_shadow_mode": {
|
||||
"node_limits": 0,
|
||||
"no_successful_request_limits": 0,
|
||||
"throughput_degradation_limits": 0
|
||||
}
|
||||
},
|
||||
"primary": {
|
||||
"primary_rejections": 0,
|
||||
"breakup_shadow_mode": {
|
||||
"node_limits": 0,
|
||||
"no_successful_request_limits": 0,
|
||||
"throughput_degradation_limits": 0
|
||||
}
|
||||
},
|
||||
"replica": {
|
||||
"replica_rejections": 0,
|
||||
"breakup_shadow_mode": {
|
||||
"node_limits": 0,
|
||||
"no_successful_request_limits": 0,
|
||||
"throughput_degradation_limits": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"last_successful_timestamp": {
|
||||
"coordinating_last_successful_request_timestamp_in_millis": 1613072107990,
|
||||
"primary_last_successful_request_timestamp_in_millis": 0,
|
||||
"replica_last_successful_request_timestamp_in_millis": 0
|
||||
},
|
||||
"indexing": {
|
||||
"coordinating_time_in_millis": 96,
|
||||
"coordinating_count": 1,
|
||||
"primary_time_in_millis": 0,
|
||||
"primary_count": 0,
|
||||
"replica_time_in_millis": 0,
|
||||
"replica_count": 0
|
||||
},
|
||||
"memory_allocation": {
|
||||
"current": {
|
||||
"current_coordinating_and_primary_bytes": 0,
|
||||
"current_replica_bytes": 0
|
||||
},
|
||||
"limit": {
|
||||
"current_coordinating_and_primary_limits_in_bytes": 51897,
|
||||
"current_replica_limits_in_bytes": 77845
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"total_rejections_breakup_shadow_mode": {
|
||||
"node_limits": 0,
|
||||
"no_successful_request_limits": 0,
|
||||
"throughput_degradation_limits": 0
|
||||
},
|
||||
"enabled": true,
|
||||
"enforced" : false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
To include all the shards with both active and previous write operations performed on them, specify the `include_all` parameter:
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
GET _nodes/_local/stats/shard_indexing_pressure?include_all
|
||||
```
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"_nodes": {
|
||||
"total": 1,
|
||||
"successful": 1,
|
||||
"failed": 0
|
||||
},
|
||||
"cluster_name": "runTask",
|
||||
"nodes": {
|
||||
"q3e1dQjFSqyPSLAgpyQlfw": {
|
||||
"timestamp": 1613072198171,
|
||||
"name": "runTask-0",
|
||||
"transport_address": "127.0.0.1:9300",
|
||||
"host": "127.0.0.1",
|
||||
"ip": "127.0.0.1:9300",
|
||||
"roles": [
|
||||
"data",
|
||||
"ingest",
|
||||
"master",
|
||||
"remote_cluster_client"
|
||||
],
|
||||
"attributes": {
|
||||
"testattr": "test"
|
||||
},
|
||||
"shard_indexing_pressure": {
|
||||
"stats": {
|
||||
"[index_name][0]": {
|
||||
"memory": {
|
||||
"current": {
|
||||
"coordinating_in_bytes": 0,
|
||||
"primary_in_bytes": 0,
|
||||
"replica_in_bytes": 0
|
||||
},
|
||||
"total": {
|
||||
"coordinating_in_bytes": 604,
|
||||
"primary_in_bytes": 604,
|
||||
"replica_in_bytes": 0
|
||||
}
|
||||
},
|
||||
"rejection": {
|
||||
"coordinating": {
|
||||
"coordinating_rejections": 0,
|
||||
"breakup": {
|
||||
"node_limits": 0,
|
||||
"no_successful_request_limits": 0,
|
||||
"throughput_degradation_limits": 0
|
||||
}
|
||||
},
|
||||
"primary": {
|
||||
"primary_rejections": 0,
|
||||
"breakup": {
|
||||
"node_limits": 0,
|
||||
"no_successful_request_limits": 0,
|
||||
"throughput_degradation_limits": 0
|
||||
}
|
||||
},
|
||||
"replica": {
|
||||
"replica_rejections": 0,
|
||||
"breakup": {
|
||||
"node_limits": 0,
|
||||
"no_successful_request_limits": 0,
|
||||
"throughput_degradation_limits": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"last_successful_timestamp": {
|
||||
"coordinating_last_successful_request_timestamp_in_millis": 1613072194656,
|
||||
"primary_last_successful_request_timestamp_in_millis": 0,
|
||||
"replica_last_successful_request_timestamp_in_millis": 0
|
||||
},
|
||||
"indexing": {
|
||||
"coordinating_time_in_millis": 145,
|
||||
"coordinating_count": 2,
|
||||
"primary_time_in_millis": 0,
|
||||
"primary_count": 0,
|
||||
"replica_time_in_millis": 0,
|
||||
"replica_count": 0
|
||||
},
|
||||
"memory_allocation": {
|
||||
"current": {
|
||||
"current_coordinating_and_primary_bytes": 0,
|
||||
"current_replica_bytes": 0
|
||||
},
|
||||
"limit": {
|
||||
"current_coordinating_and_primary_limits_in_bytes": 51897,
|
||||
"current_replica_limits_in_bytes": 77845
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"total_rejections_breakup": {
|
||||
"node_limits": 0,
|
||||
"no_successful_request_limits": 0,
|
||||
"throughput_degradation_limits": 0
|
||||
},
|
||||
"enabled": true,
|
||||
"enforced": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
To get only all the top-level aggregated stats, specify the `top` parameter (skips the per-shard stats).
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
GET _nodes/_local/stats/shard_indexing_pressure?top
|
||||
```
|
||||
|
||||
If `enforced` is `true`:
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"_nodes": {
|
||||
"total": 1,
|
||||
"successful": 1,
|
||||
"failed": 0
|
||||
},
|
||||
"cluster_name": "runTask",
|
||||
"nodes": {
|
||||
"q3e1dQjFSqyPSLAgpyQlfw": {
|
||||
"timestamp": 1613072382719,
|
||||
"name": "runTask-0",
|
||||
"transport_address": "127.0.0.1:9300",
|
||||
"host": "127.0.0.1",
|
||||
"ip": "127.0.0.1:9300",
|
||||
"roles": [
|
||||
"data",
|
||||
"ingest",
|
||||
"master",
|
||||
"remote_cluster_client"
|
||||
],
|
||||
"attributes": {
|
||||
"testattr": "test"
|
||||
},
|
||||
"shard_indexing_pressure": {
|
||||
"stats": {},
|
||||
"total_rejections_breakup": {
|
||||
"node_limits": 0,
|
||||
"no_successful_request_limits": 0,
|
||||
"throughput_degradation_limits": 0
|
||||
},
|
||||
"enabled": true,
|
||||
"enforced": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
If `enforced` is `false`:
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"_nodes": {
|
||||
"total": 1,
|
||||
"successful": 1,
|
||||
"failed": 0
|
||||
},
|
||||
"cluster_name": "runTask",
|
||||
"nodes": {
|
||||
"q3e1dQjFSqyPSLAgpyQlfw": {
|
||||
"timestamp": 1613072382719,
|
||||
"name": "runTask-0",
|
||||
"transport_address": "127.0.0.1:9300",
|
||||
"host": "127.0.0.1",
|
||||
"ip": "127.0.0.1:9300",
|
||||
"roles": [
|
||||
"data",
|
||||
"ingest",
|
||||
"master",
|
||||
"remote_cluster_client"
|
||||
],
|
||||
"attributes": {
|
||||
"testattr": "test"
|
||||
},
|
||||
"shard_indexing_pressure": {
|
||||
"stats": {},
|
||||
"total_rejections_breakup_shadow_mode": {
|
||||
"node_limits": 0,
|
||||
"no_successful_request_limits": 0,
|
||||
"throughput_degradation_limits": 0
|
||||
},
|
||||
"enabled": true,
|
||||
"enforced" : false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
To get the shard-level breakup of rejections for every node (only includes shards with active write operations):
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
GET _nodes/stats/shard_indexing_pressure
|
||||
```
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"_nodes": {
|
||||
"total": 1,
|
||||
"successful": 1,
|
||||
"failed": 0
|
||||
},
|
||||
"cluster_name": "runTask",
|
||||
"nodes": {
|
||||
"q3e1dQjFSqyPSLAgpyQlfw": {
|
||||
"timestamp": 1613072382719,
|
||||
"name": "runTask-0",
|
||||
"transport_address": "127.0.0.1:9300",
|
||||
"host": "127.0.0.1",
|
||||
"ip": "127.0.0.1:9300",
|
||||
"roles": [
|
||||
"data",
|
||||
"ingest",
|
||||
"master",
|
||||
"remote_cluster_client"
|
||||
],
|
||||
"attributes": {
|
||||
"testattr": "test"
|
||||
},
|
||||
"shard_indexing_pressure": {
|
||||
"stats": {},
|
||||
"total_rejections_breakup": {
|
||||
"node_limits": 0,
|
||||
"no_successful_request_limits": 0,
|
||||
"throughput_degradation_limits": 0
|
||||
},
|
||||
"enabled": true,
|
||||
"enforced": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
|
@ -19,68 +19,53 @@ require "pathname"
|
|||
|
||||
module Jekyll::LinkChecker
|
||||
|
||||
##
|
||||
# The collection that will get stores as the output
|
||||
|
||||
@urls = {}
|
||||
|
||||
##
|
||||
# Pattern to identify documents that should be excluded based on their URL
|
||||
@excluded_paths = /(\.(css|js|json|map|xml|txt|yml)$|\/version-selector\.tpl$)/i.freeze
|
||||
|
||||
@excluded_paths = /(\.(css|js|json|map|xml|txt|yml)$)/i.freeze
|
||||
|
||||
##
|
||||
# Pattern to identify certain HTML tags whose content should be excluded from indexing
|
||||
|
||||
@href_matcher = /<a[^>]+href=(['"])(.+?)\1/im.freeze
|
||||
|
||||
##
|
||||
# Pattern to check for external URLs
|
||||
|
||||
@external_matcher = /^https?:\/\//.freeze
|
||||
|
||||
##
|
||||
# List of domains to ignore
|
||||
@ignored_domains = %w[localhost]
|
||||
|
||||
##
|
||||
# Pattern of local paths to ignore
|
||||
@ignored_paths = /(^\/javadocs\/)/.freeze
|
||||
|
||||
##
|
||||
# Pattern to exclude when adding the `index.html` suffix to paths
|
||||
@need_no_suffix = /\.(?!html)[^\/]+$/.freeze
|
||||
|
||||
# Valid response codes for successful links
|
||||
@success_codes = %w[200 302]
|
||||
|
||||
##
|
||||
# Questionable response codes for successful links
|
||||
@questionable_codes = %w[301 403 429]
|
||||
|
||||
##
|
||||
# Holds the list of failures
|
||||
@failures = []
|
||||
|
||||
##
|
||||
# Driven by environment variables, it indicates a need to check external links
|
||||
@check_external_links
|
||||
|
||||
##
|
||||
# Driven by environment variables, it indicates the need to fail the build for dead links
|
||||
@should_build_fatally
|
||||
|
||||
|
||||
##
|
||||
# Initializes the singleton by recording the site
|
||||
|
||||
# return [void]
|
||||
def self.init(site)
|
||||
@site = site
|
||||
@urls = {}
|
||||
@failures = []
|
||||
end
|
||||
|
||||
##
|
||||
# Processes a Document or Page and adds the links to a collection
|
||||
# It also checks for anchors to parts of the same page/doc
|
||||
|
||||
# It also checks for anchors that link to parts of the same page/doc
|
||||
# return [void]
|
||||
def self.process(page)
|
||||
return if @excluded_paths.match(page.path)
|
||||
|
||||
|
|
@ -98,9 +83,8 @@ module Jekyll::LinkChecker
|
|||
end
|
||||
end
|
||||
|
||||
##
|
||||
# Saves the collection as a JSON file
|
||||
|
||||
# Verifies the validity of all the destinations gathered in @urls
|
||||
# return [void]
|
||||
def self.verify(site)
|
||||
if ENV.key?('JEKYLL_CHECK_EXTERNAL_LINKS')
|
||||
@check_external_links = true
|
||||
|
|
@ -132,9 +116,9 @@ module Jekyll::LinkChecker
|
|||
end
|
||||
end
|
||||
|
||||
##
|
||||
# Check if URL is accessible
|
||||
|
||||
# Check if an internal or external URL is accessible
|
||||
# @param url [String] the url to check
|
||||
# @return [Boolean]
|
||||
def self.check(url)
|
||||
match = @base_url_matcher.match(url)
|
||||
unless match.nil?
|
||||
|
|
@ -149,9 +133,9 @@ module Jekyll::LinkChecker
|
|||
return self.check_internal(url)
|
||||
end
|
||||
|
||||
##
|
||||
# Check if an external URL is accessible by making a HEAD call
|
||||
|
||||
# @param url [String] the url to check
|
||||
# @return [Boolean]
|
||||
def self.check_external(url)
|
||||
uri = URI(url)
|
||||
return true if @ignored_domains.include? uri.host
|
||||
|
|
@ -172,61 +156,62 @@ module Jekyll::LinkChecker
|
|||
end
|
||||
end
|
||||
|
||||
##
|
||||
# Check if an internal link is accessible
|
||||
|
||||
# @param url [String] the url to check
|
||||
# @return [Boolean]
|
||||
def self.check_internal(url)
|
||||
return true if @ignored_paths =~ url
|
||||
|
||||
path, hash = url.split('#')
|
||||
|
||||
unless path.end_with? 'index.html'
|
||||
path << '/' unless path.end_with? '/'
|
||||
path << 'index.html' unless path.end_with? 'index.html'
|
||||
if @need_no_suffix =~ path
|
||||
filename = File.join(@site.config["destination"], path)
|
||||
return File.file?(filename)
|
||||
else
|
||||
unless path.end_with? 'index.html'
|
||||
path << '/' unless path.end_with? '/'
|
||||
path << 'index.html' unless path.end_with? 'index.html'
|
||||
end
|
||||
|
||||
filename = File.join(@site.config["destination"], path)
|
||||
|
||||
return false unless File.file?(filename)
|
||||
|
||||
content = File.read(filename)
|
||||
unless content.include? "<title>Redirecting"
|
||||
return true if hash.nil? || hash.empty?
|
||||
return !(content =~ /<[a-z0-9-]+[^>]+id="#{hash}"/i).nil?
|
||||
end
|
||||
|
||||
match = content.match(@href_matcher)
|
||||
if match.nil?
|
||||
puts "LinkChecker: [Warning] Cannot check #{url} due to an unfollowable redirect"
|
||||
return true
|
||||
end
|
||||
|
||||
redirect = match[2]
|
||||
redirect << '#' + hash unless hash.nil? || hash.empty?
|
||||
return self.check(redirect)
|
||||
end
|
||||
|
||||
filename = File.join(@site.config["destination"], path)
|
||||
|
||||
return false unless File.file?(filename)
|
||||
|
||||
content = File.read(filename)
|
||||
unless content.include? "<title>Redirecting"
|
||||
return true if hash.nil? || hash.empty?
|
||||
return !(content =~ /<[a-z0-9-]+[^>]+id="#{hash}"/i).nil?
|
||||
end
|
||||
|
||||
match = content.match(@href_matcher)
|
||||
if match.nil?
|
||||
puts "LinkChecker: [Warning] Cannot check #{url} due to an unfollowable redirect"
|
||||
return true
|
||||
end
|
||||
|
||||
redirect = match[2]
|
||||
redirect << '#' + hash unless hash.nil? || hash.empty?
|
||||
return self.check(redirect)
|
||||
end
|
||||
end
|
||||
|
||||
# Before any Document or Page is processed, initialize the LinkChecker
|
||||
|
||||
Jekyll::Hooks.register :site, :pre_render do |site|
|
||||
Jekyll::LinkChecker.init(site)
|
||||
end
|
||||
|
||||
# Process a Page as soon as its content is ready
|
||||
|
||||
Jekyll::Hooks.register :pages, :post_convert do |page|
|
||||
Jekyll::LinkChecker.process(page)
|
||||
end
|
||||
|
||||
# Process a Document as soon as its content is ready
|
||||
|
||||
Jekyll::Hooks.register :documents, :post_convert do |document|
|
||||
Jekyll::LinkChecker.process(document)
|
||||
end
|
||||
|
||||
# Verify gathered links after Jekyll is done writing all its stuff
|
||||
|
||||
Jekyll::Hooks.register :site, :post_write do |site|
|
||||
Jekyll::LinkChecker.verify(site)
|
||||
end
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require "jekyll/hooks"
|
||||
require "jekyll/document"
|
||||
require "json"
|
||||
|
||||
##
|
||||
# This singleton facilitates production of an indexable JSON representation of the content to populate a data source
|
||||
# to provide search functionality.
|
||||
|
||||
module Jekyll::ContentIndexer
|
||||
|
||||
##
|
||||
# The collection that will get stores as the output
|
||||
|
||||
@data = []
|
||||
|
||||
##
|
||||
# Pattern to identify documents that should be excluded based on their URL
|
||||
|
||||
@excluded_paths = /\.(css|js|json|map|xml|txt|yml)$/i.freeze
|
||||
|
||||
##
|
||||
# Pattern to identify block HTML tags (not comprehensive)
|
||||
|
||||
@html_block_tags = /\s*<[?\/]?(article|blockquote|d[dlt]|div|fieldset|form|h|li|main|nav|[ou]l|p|section|table|t[rd]).*?>\s*/im.freeze
|
||||
|
||||
##
|
||||
# Pattern to identify certain HTML tags whose content should be excluded from indexing
|
||||
|
||||
@html_excluded_tags = /\s*<(head|style|script|h1).*?>.*?<\/\1>/im.freeze
|
||||
|
||||
##
|
||||
# Initializes the singleton by recording the site
|
||||
|
||||
def self.init(site)
|
||||
@site = site
|
||||
end
|
||||
|
||||
##
|
||||
# Processes a Document or Page and adds it to the collection
|
||||
|
||||
def self.add(page)
|
||||
return if @excluded_paths.match(page.url)
|
||||
|
||||
content = page.content
|
||||
.gsub(@html_excluded_tags, ' ') # Strip certain HTML blocks
|
||||
.gsub(@html_block_tags, "\n") # Strip some block HTML tags, replacing with newline
|
||||
.gsub(/\s*<[?\/!]?[a-z]+.*?>\s*/im, ' ') # Strip all remaining HTML tags
|
||||
.gsub(/\s*[\r\n]+\s*/, "\n") # Clean line-breaks
|
||||
.gsub(/\s{2,}/, ' ') # Trim long spaces
|
||||
.gsub(/\s+([.:;,)!\]?])/, '\1') # Remove spaces before some punctuations
|
||||
.strip # Trim leading and tailing whitespaces
|
||||
|
||||
return if content.empty?
|
||||
|
||||
url = @site.config["baseurl"] + page.url
|
||||
|
||||
# Produce a breadcrumb
|
||||
ancestors = []
|
||||
if page.instance_of?(Jekyll::Document)
|
||||
ancestors.push(@site.config.dig("just_the_docs", "collections", page.collection&.label, "name"))
|
||||
end
|
||||
|
||||
ancestors.push(page.data["grand_parent"]) unless
|
||||
page.data["grand_parent"].nil? ||
|
||||
page.data["grand_parent"]&.empty? ||
|
||||
ancestors.include?(page.data["grand_parent"]) # Make sure collection name is not added
|
||||
|
||||
ancestors.push(page.data["parent"]) unless
|
||||
page.data["parent"].nil? ||
|
||||
page.data["parent"]&.empty? ||
|
||||
ancestors.include?(page.data["parent"]) # Make sure collection name is not added
|
||||
|
||||
data = {
|
||||
url: url,
|
||||
title: page.data["title"],
|
||||
content: content,
|
||||
ancestors: ancestors,
|
||||
type: "DOCS"
|
||||
}
|
||||
|
||||
@data.push(data)
|
||||
end
|
||||
|
||||
##
|
||||
# Saves the collection as a JSON file
|
||||
|
||||
def self.save
|
||||
File.open(File.join(@site.config["destination"], "search-index.json"), 'w') do |f|
|
||||
f.puts JSON.pretty_generate(@data)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Before any Document or Page is processed, initialize the ContentIndexer
|
||||
|
||||
Jekyll::Hooks.register :site, :pre_render do |site|
|
||||
Jekyll::ContentIndexer.init(site)
|
||||
end
|
||||
|
||||
# Process a Page as soon as its content is ready
|
||||
|
||||
Jekyll::Hooks.register :pages, :post_convert do |page|
|
||||
Jekyll::ContentIndexer.add(page)
|
||||
end
|
||||
|
||||
# Process a Document as soon as its content is ready
|
||||
|
||||
Jekyll::Hooks.register :documents, :post_convert do |document|
|
||||
Jekyll::ContentIndexer.add(document)
|
||||
end
|
||||
|
||||
# Save the produced collection after Jekyll is done writing all its stuff
|
||||
|
||||
Jekyll::Hooks.register :site, :post_write do |_|
|
||||
Jekyll::ContentIndexer.save()
|
||||
end
|
||||
|
|
@ -0,0 +1,391 @@
|
|||
---
|
||||
layout: default
|
||||
title: API
|
||||
nav_order: 50
|
||||
---
|
||||
|
||||
# Cross-cluster replication API
|
||||
|
||||
Use these replication operations to programmatically manage cross-cluster replication.
|
||||
|
||||
#### Table of contents
|
||||
- TOC
|
||||
{:toc}
|
||||
|
||||
## Start replication
|
||||
Introduced 1.1
|
||||
{: .label .label-purple }
|
||||
|
||||
Initiate replication of an index from the leader cluster to the follower cluster. Send this request to the follower cluster.
|
||||
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
PUT /_plugins/_replication/<follower-index>/_start
|
||||
{
|
||||
"leader_alias":"<connection-alias-name>",
|
||||
"leader_index":"<index-name>",
|
||||
"use_roles":{
|
||||
"leader_cluster_role":"<role-name>",
|
||||
"follower_cluster_role":"<role-name>"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Specify the following options:
|
||||
|
||||
Options | Description | Type | Required
|
||||
:--- | :--- |:--- |:--- |
|
||||
`leader_alias` | The name of the cross-cluster connection. You define this alias when you [set up a cross-cluster connection]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/#set-up-a-cross-cluster-connection). | `string` | Yes
|
||||
`leader_index` | The index on the leader cluster that you want to replicate. | `string` | Yes
|
||||
`use_roles` | The roles to use for all subsequent backend replication tasks between the indices. Specify a `leader_cluster_role` and `follower_cluster_role`. See [Map the leader and follower cluster roles]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). | `string` | If security plugin is enabled
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"acknowledged": true
|
||||
}
|
||||
```
|
||||
|
||||
## Stop replication
|
||||
Introduced 1.1
|
||||
{: .label .label-purple }
|
||||
|
||||
Terminates replication and converts the follower index to a standard index. Send this request to the follower cluster.
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
POST /_plugins/_replication/<follower-index>/_stop
|
||||
{}
|
||||
```
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"acknowledged": true
|
||||
}
|
||||
```
|
||||
|
||||
## Pause replication
|
||||
Introduced 1.1
|
||||
{: .label .label-purple }
|
||||
|
||||
Pauses replication of the leader index. Send this request to the follower cluster.
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
POST /_plugins/_replication/<follower-index>/_pause
|
||||
{}
|
||||
```
|
||||
|
||||
You can't resume replication after it's been paused for more than 12 hours. You must [stop replication]({{site.url}}{{site.baseurl}}/replication-plugin/api/#stop-replication), delete the follower index, and restart replication of the leader.
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"acknowledged": true
|
||||
}
|
||||
```
|
||||
|
||||
## Resume replication
|
||||
Introduced 1.1
|
||||
{: .label .label-purple }
|
||||
|
||||
Resumes replication of the leader index. Send this request to the follower cluster.
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
POST /_plugins/_replication/<follower-index>/_resume
|
||||
{}
|
||||
```
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"acknowledged": true
|
||||
}
|
||||
```
|
||||
|
||||
## Get replication status
|
||||
Introduced 1.1
|
||||
{: .label .label-purple }
|
||||
|
||||
Gets the status of index replication. Possible statuses are `SYNCING`, `BOOTSTRAPING`, `PAUSED`, and `REPLICATION NOT IN PROGRESS`. Use the syncing details to measure replication lag. Send this request to the follower cluster.
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
GET /_plugins/_replication/<follower-index>/_status
|
||||
```
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"status" : "SYNCING",
|
||||
"reason" : "User initiated",
|
||||
"leader_alias" : "my-connection-name",
|
||||
"leader_index" : "leader-01",
|
||||
"follower_index" : "follower-01",
|
||||
"syncing_details" : {
|
||||
"leader_checkpoint" : 19,
|
||||
"follower_checkpoint" : 19,
|
||||
"seq_no" : 0
|
||||
}
|
||||
}
|
||||
```
|
||||
To include shard replication details in the response, add the `&verbose=true` parameter.
|
||||
|
||||
The leader and follower checkpoint values begin as negative integers and reflect the shard count (-1 for one shard, -5 for five shards, and so on). The values increment toward positive integers with each change that you make. For example, when you make a change on the leader index, the `leader_checkpoint` becomes `0`. The `follower_checkpoint` is initially still `-1` until the follower index pulls the change from the leader, at which point it increments to `0`. If the values are the same, it means the indices are fully synced.
|
||||
|
||||
## Get leader cluster stats
|
||||
Introduced 1.1
|
||||
{: .label .label-purple }
|
||||
|
||||
Gets information about replicated leader indices on a specified cluster.
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
GET /_plugins/_replication/leader_stats
|
||||
```
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"num_replicated_indices": 2,
|
||||
"operations_read": 15,
|
||||
"translog_size_bytes": 1355,
|
||||
"operations_read_lucene": 0,
|
||||
"operations_read_translog": 15,
|
||||
"total_read_time_lucene_millis": 0,
|
||||
"total_read_time_translog_millis": 659,
|
||||
"bytes_read": 1000,
|
||||
"index_stats":{
|
||||
"leader-index-1":{
|
||||
"operations_read": 7,
|
||||
"translog_size_bytes": 639,
|
||||
"operations_read_lucene": 0,
|
||||
"operations_read_translog": 7,
|
||||
"total_read_time_lucene_millis": 0,
|
||||
"total_read_time_translog_millis": 353,
|
||||
"bytes_read":466
|
||||
},
|
||||
"leader-index-2":{
|
||||
"operations_read": 8,
|
||||
"translog_size_bytes": 716,
|
||||
"operations_read_lucene": 0,
|
||||
"operations_read_translog": 8,
|
||||
"total_read_time_lucene_millis": 0,
|
||||
"total_read_time_translog_millis": 306,
|
||||
"bytes_read": 534
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Get follower cluster stats
|
||||
Introduced 1.1
|
||||
{: .label .label-purple }
|
||||
|
||||
Gets information about follower (syncing) indices on a specified cluster.
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
GET /_plugins/_replication/follower_stats
|
||||
```
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"num_syncing_indices": 2,
|
||||
"num_bootstrapping_indices": 0,
|
||||
"num_paused_indices": 0,
|
||||
"num_failed_indices": 0,
|
||||
"num_shard_tasks": 2,
|
||||
"num_index_tasks": 2,
|
||||
"operations_written": 3,
|
||||
"operations_read": 3,
|
||||
"failed_read_requests": 0,
|
||||
"throttled_read_requests": 0,
|
||||
"failed_write_requests": 0,
|
||||
"throttled_write_requests": 0,
|
||||
"follower_checkpoint": 1,
|
||||
"leader_checkpoint": 1,
|
||||
"total_write_time_millis": 2290,
|
||||
"index_stats":{
|
||||
"follower-index-1":{
|
||||
"operations_written": 2,
|
||||
"operations_read": 2,
|
||||
"failed_read_requests": 0,
|
||||
"throttled_read_requests": 0,
|
||||
"failed_write_requests": 0,
|
||||
"throttled_write_requests": 0,
|
||||
"follower_checkpoint": 1,
|
||||
"leader_checkpoint": 1,
|
||||
"total_write_time_millis": 1355
|
||||
},
|
||||
"follower-index-2":{
|
||||
"operations_written": 1,
|
||||
"operations_read": 1,
|
||||
"failed_read_requests": 0,
|
||||
"throttled_read_requests": 0,
|
||||
"failed_write_requests": 0,
|
||||
"throttled_write_requests": 0,
|
||||
"follower_checkpoint": 0,
|
||||
"leader_checkpoint": 0,
|
||||
"total_write_time_millis": 935
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Get auto-follow stats
|
||||
Introduced 1.1
|
||||
{: .label .label-purple }
|
||||
|
||||
Gets information about auto-follow activity and any replication rules configured on the specified cluster.
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
GET /_plugins/_replication/autofollow_stats
|
||||
```
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"num_success_start_replication": 2,
|
||||
"num_failed_start_replication": 0,
|
||||
"num_failed_leader_calls": 0,
|
||||
"failed_indices":[
|
||||
|
||||
],
|
||||
"autofollow_stats":[
|
||||
{
|
||||
"name":"my-replication-rule",
|
||||
"pattern":"movies*",
|
||||
"num_success_start_replication": 2,
|
||||
"num_failed_start_replication": 0,
|
||||
"num_failed_leader_calls": 0,
|
||||
"failed_indices":[
|
||||
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Update settings
|
||||
Introduced 1.1
|
||||
{: .label .label-purple }
|
||||
|
||||
Updates settings on the follower index.
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
PUT /_plugins/_replication/<follower-index>/_update
|
||||
{
|
||||
"settings":{
|
||||
"index.number_of_shards": 4,
|
||||
"index.number_of_replicas": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"acknowledged": true
|
||||
}
|
||||
```
|
||||
|
||||
## Create replication rule
|
||||
Introduced 1.1
|
||||
{: .label .label-purple }
|
||||
|
||||
Automatically starts replication on indices matching a specified pattern. If a new index on the leader cluster matches the pattern, OpenSearch automatically creates a follower index and begins replication. You can also use this API to update existing replication rules.
|
||||
|
||||
Send this request to the follower cluster.
|
||||
|
||||
Make sure to note the names of all auto-follow patterns after you create them. The replication plugin currently does not include an API operation to retrieve a list of existing patterns.
|
||||
{: .tip }
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
POST /_plugins/_replication/_autofollow
|
||||
{
|
||||
"leader_alias" : "<connection-alias-name>",
|
||||
"name": "<auto-follow-pattern-name>",
|
||||
"pattern": "<pattern>",
|
||||
"use_roles":{
|
||||
"leader_cluster_role": "<role-name>",
|
||||
"follower_cluster_role": "<role-name>"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Specify the following options:
|
||||
|
||||
Options | Description | Type | Required
|
||||
:--- | :--- |:--- |:--- |
|
||||
`leader_alias` | The name of the cross-cluster connection. You define this alias when you [set up a cross-cluster connection]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/#set-up-a-cross-cluster-connection). | `string` | Yes
|
||||
`name` | A name for the auto-follow pattern. | `string` | Yes
|
||||
`pattern` | An array of index patterns to match against indices in the specified leader cluster. Supports wildcard characters. For example, `leader-*`. | `string` | Yes
|
||||
`use_roles` | The roles to use for all subsequent backend replication tasks between the indices. Specify a `leader_cluster_role` and `follower_cluster_role`. See [Map the leader and follower cluster roles]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). | `string` | If security plugin is enabled
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"acknowledged": true
|
||||
}
|
||||
```
|
||||
|
||||
## Delete replication rule
|
||||
Introduced 1.1
|
||||
{: .label .label-purple }
|
||||
|
||||
Deletes the specified replication rule. This operation prevents any new indices from being replicated but does not stop existing replication that the rule has already initiated. Replicated indices remain read-only until you stop replication.
|
||||
|
||||
Send this request to the follower cluster.
|
||||
|
||||
#### Request
|
||||
|
||||
```json
|
||||
DELETE /_plugins/_replication/_autofollow
|
||||
{
|
||||
"leader_alias" : "<connection-alias-name>",
|
||||
"name": "<auto-follow-pattern-name>",
|
||||
}
|
||||
```
|
||||
|
||||
Specify the following options:
|
||||
|
||||
Options | Description | Type | Required
|
||||
:--- | :--- |:--- |:--- |
|
||||
`leader_alias` | The name of the cross-cluster connection. You define this alias when you [set up a cross-cluster connection]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/#set-up-a-cross-cluster-connection). | `string` | Yes
|
||||
`name` | The name of the pattern. | `string` | Yes
|
||||
|
||||
#### Sample response
|
||||
|
||||
```json
|
||||
{
|
||||
"acknowledged": true
|
||||
}
|
||||
```
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
---
|
||||
layout: default
|
||||
title: Auto-follow
|
||||
nav_order: 20
|
||||
has_children: false
|
||||
|
||||
---
|
||||
|
||||
# Auto-follow for cross-cluster replication
|
||||
|
||||
Auto-follow lets you automatically replicate indices created on the leader cluster based on matching patterns. When you create an index on the leader cluster with a name that matches a specified pattern (for example, `index-01*`), a corresponding follower index is automatically created on the follower cluster.
|
||||
|
||||
You can configure multiple replication rules for a single cluster. The patterns currently only support wildcard matching.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
You need to [set up a cross-cluster connection]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/#set-up-a-cross-cluster-connection) between two clusters before you can enable auto-follow.
|
||||
|
||||
## Permissions
|
||||
|
||||
If the security plugin is enabled, non-admin users need to be mapped to the appropriate permissions in order to perform replication actions. For index and cluster-level permissions requirements, see [Cross-cluster replication permissions]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/).
|
||||
|
||||
## Get started with auto-follow
|
||||
|
||||
Replication rules are a collection of patterns that you create against a single remote cluster. When you create a replication rule, it automatically starts replicating any *new* indices that match the pattern, but does not replicate matching indices that were previously created.
|
||||
|
||||
Create a replication rule on the follower cluster:
|
||||
|
||||
```bash
|
||||
curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/_autofollow?pretty' -d '
|
||||
{
|
||||
"leader_alias" : "my-connection-alias",
|
||||
"name": "my-replication-rule",
|
||||
"pattern": "movies*",
|
||||
"use_roles":{
|
||||
"leader_cluster_role": "all_access",
|
||||
"follower_cluster_role": "all_access"
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
If the security plugin is disabled, you can leave out the `use_roles` parameter. If it's enabled, however, you need to specify the leader and follower cluster roles that OpenSearch uses to authenticate requests. This example uses `all_access` for simplicity, but we recommend creating a replication user on each cluster and [mapping it accordingly]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles).
|
||||
{: .tip }
|
||||
|
||||
To test the rule, create a matching index on the leader cluster:
|
||||
|
||||
```bash
|
||||
curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/movies-0001?pretty'
|
||||
```
|
||||
|
||||
And confirm its replica shows up on the follower cluster:
|
||||
|
||||
```bash
|
||||
curl -XGET -u 'admin:admin' -k 'https://localhost:9200/_cat/indices?v'
|
||||
```
|
||||
|
||||
It might take several seconds for the index to appear.
|
||||
|
||||
```bash
|
||||
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
|
||||
yellow open movies-0001 kHOxYYHxRMeszLjTD9rvSQ 1 1 0 0 208b 208b
|
||||
```
|
||||
|
||||
## Retrieve replication rules
|
||||
|
||||
To retrieve a list of existing replication rules configured on a cluster, send the following request:
|
||||
|
||||
```bash
|
||||
curl -XGET -u 'admin:admin' -k 'https://localhost:9200/_plugins/_replication/autofollow_stats'
|
||||
|
||||
{
|
||||
"num_success_start_replication": 1,
|
||||
"num_failed_start_replication": 0,
|
||||
"num_failed_leader_calls": 0,
|
||||
"failed_indices":[
|
||||
|
||||
],
|
||||
"autofollow_stats":[
|
||||
{
|
||||
"name":"my-replication-rule",
|
||||
"pattern":"movies*",
|
||||
"num_success_start_replication": 1,
|
||||
"num_failed_start_replication": 0,
|
||||
"num_failed_leader_calls": 0,
|
||||
"failed_indices":[
|
||||
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Delete a replication rule
|
||||
|
||||
To delete a replication rule, send the following request:
|
||||
|
||||
```bash
|
||||
curl -XDELETE -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/_autofollow?pretty' -d '
|
||||
{
|
||||
"leader_alias" : "my-conection-alias",
|
||||
"name": "my-replication-rule"
|
||||
}'
|
||||
```
|
||||
|
||||
OpenSearch stops replicating *new* indices that match the pattern, but existing indices that the rule previously created remain read-only and continue to replicate. If you need to stop existing replication activity and open the indices up for writes, use the [stop replication API operation]({{site.url}}{{site.baseurl}}/replication-plugin/api/#stop-replication).
|
||||
|
||||
|
|
@ -0,0 +1,293 @@
|
|||
---
|
||||
layout: default
|
||||
title: Get started
|
||||
nav_order: 10
|
||||
---
|
||||
|
||||
# Get started with cross-cluster replication
|
||||
|
||||
With cross-cluster replication, you index data to a leader index, and OpenSearch replicates that data to one or more read-only follower indices. All subsequent operations on the leader are replicated on the follower, such as creating, updating, or deleting documents.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Cross-cluster replication has the following prerequisites:
|
||||
- Both the leader and follower cluster must have the replication plugin installed.
|
||||
- If you've overridden `node.roles` in `opensearch.yml` on the follower cluster, make sure it also includes the `remote_cluster_client` role:
|
||||
|
||||
```yaml
|
||||
node.roles: [<other_roles>, remote_cluster_client]
|
||||
```
|
||||
|
||||
## Permissions
|
||||
|
||||
Make sure the security plugin is either enabled on both clusters or disabled on both clusters. If you disabled the security plugin, you can skip this section. However, we strongly recommend enabling the security plugin in production scenarios.
|
||||
|
||||
If the security plugin is enabled, non-admin users need to be mapped to the appropriate permissions in order to perform replication actions. For index and cluster-level permissions requirements, see [Cross-cluster replication permissions]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/).
|
||||
|
||||
In addition, verify and add the Distinguished names (DNs) of each follower cluster node on the leader cluster to allow connections from the followers to the leader.
|
||||
|
||||
First, get the node's DN from each follower cluster:
|
||||
|
||||
```bash
|
||||
curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_security/api/ssl/certs?pretty'
|
||||
|
||||
{
|
||||
"transport_certificates_list": [
|
||||
{
|
||||
"issuer_dn" : "CN=Test,OU=Server CA 1B,O=Test,C=US",
|
||||
"subject_dn" : "CN=follower.test.com", # To be added under leader's nodes_dn configuration
|
||||
"not_before" : "2021-11-12T00:00:00Z",
|
||||
"not_after" : "2022-12-11T23:59:59Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Then verify that it's part of the leader cluster configuration in `opensearch.yml`, otherwise add it under the following setting:
|
||||
|
||||
```yaml
|
||||
plugins.security.nodes_dn:
|
||||
- "CN=*.leader.com, OU=SSL, O=Test, L=Test, C=DE" # Already part of the configuration
|
||||
- "CN=follower.test.com" # From the above response from follower
|
||||
```
|
||||
## Example setup
|
||||
|
||||
Save this sample file as `docker-compose.yml` and run `docker-compose up` to start two single-node clusters on the same network:
|
||||
|
||||
```yml
|
||||
version: '3'
|
||||
services:
|
||||
replication-node1:
|
||||
image: opensearchproject/opensearch:{{site.opensearch_version}}
|
||||
container_name: replication-node1
|
||||
environment:
|
||||
- cluster.name=leader-cluster
|
||||
- discovery.type=single-node
|
||||
- bootstrap.memory_lock=true
|
||||
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
volumes:
|
||||
- opensearch-data2:/usr/share/opensearch/data
|
||||
ports:
|
||||
- 9201:9200
|
||||
- 9700:9600 # required for Performance Analyzer
|
||||
networks:
|
||||
- opensearch-net
|
||||
replication-node2:
|
||||
image: opensearchproject/opensearch:{{site.opensearch_version}}
|
||||
container_name: replication-node2
|
||||
environment:
|
||||
- cluster.name=follower-cluster
|
||||
- discovery.type=single-node
|
||||
- bootstrap.memory_lock=true
|
||||
- "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
|
||||
ulimits:
|
||||
memlock:
|
||||
soft: -1
|
||||
hard: -1
|
||||
volumes:
|
||||
- opensearch-data1:/usr/share/opensearch/data
|
||||
ports:
|
||||
- 9200:9200
|
||||
- 9600:9600 # required for Performance Analyzer
|
||||
networks:
|
||||
- opensearch-net
|
||||
|
||||
volumes:
|
||||
opensearch-data1:
|
||||
opensearch-data2:
|
||||
|
||||
networks:
|
||||
opensearch-net:
|
||||
```
|
||||
|
||||
After the clusters start, verify the names of each:
|
||||
|
||||
```bash
|
||||
curl -XGET -u 'admin:admin' -k 'https://localhost:9201'
|
||||
{
|
||||
"cluster_name" : "leader-cluster",
|
||||
...
|
||||
}
|
||||
|
||||
curl -XGET -u 'admin:admin' -k 'https://localhost:9200'
|
||||
{
|
||||
"cluster_name" : "follower-cluster",
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
For this example, use port 9201 (`replication-node1`) as the leader and port 9200 (`replication-node2`) as the follower cluster.
|
||||
|
||||
To get the IP address for the leader cluster, first identify its container ID:
|
||||
|
||||
```bash
|
||||
docker ps
|
||||
CONTAINER ID IMAGE PORTS NAMES
|
||||
3b8cdc698be5 opensearchproject/opensearch:{{site.opensearch_version}} 0.0.0.0:9200->9200/tcp, 0.0.0.0:9600->9600/tcp, 9300/tcp replication-node2
|
||||
731f5e8b0f4b opensearchproject/opensearch:{{site.opensearch_version}} 9300/tcp, 0.0.0.0:9201->9200/tcp, 0.0.0.0:9700->9600/tcp replication-node1
|
||||
```
|
||||
|
||||
Then get that container's IP address:
|
||||
|
||||
```bash
|
||||
docker inspect --format='{% raw %}{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}{% endraw %}' 731f5e8b0f4b
|
||||
172.22.0.3
|
||||
```
|
||||
|
||||
## Set up a cross-cluster connection
|
||||
|
||||
Cross-cluster replication follows a "pull" model, so most changes occur on the follower cluster, not the leader cluster.
|
||||
|
||||
On the follower cluster, add the IP address (with port 9300) for each seed node. Because this is a single-node cluster, you only have one seed node. Provide a descriptive name for the connection, which you'll use in the request to start replication:
|
||||
|
||||
```bash
|
||||
curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_cluster/settings?pretty' -d '
|
||||
{
|
||||
"persistent": {
|
||||
"cluster": {
|
||||
"remote": {
|
||||
"my-connection-alias": {
|
||||
"seeds": ["172.22.0.3:9300"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
## Start replication
|
||||
|
||||
To get started, create an index called `leader-01` on the leader cluster:
|
||||
|
||||
```bash
|
||||
curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/leader-01?pretty'
|
||||
```
|
||||
|
||||
Then start replication from the follower cluster. In the request body, provide the connection name and leader index that you want to replicate, along with the security roles you want to use:
|
||||
|
||||
```bash
|
||||
curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_start?pretty' -d '
|
||||
{
|
||||
"leader_alias": "my-connection-alias",
|
||||
"leader_index": "leader-01",
|
||||
"use_roles":{
|
||||
"leader_cluster_role": "all_access",
|
||||
"follower_cluster_role": "all_access"
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
If the security plugin is disabled, omit the `use_roles` parameter. If it's enabled, however, you must specify the leader and follower cluster roles that OpenSearch will use to authenticate the request. This example uses `all_access` for simplicity, but we recommend creating a replication user on each cluster and [mapping it accordingly]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles).
|
||||
{: .tip }
|
||||
|
||||
This command creates an identical read-only index named `follower-01` on the follower cluster that continuously stays updated with changes to the `leader-01` index on the leader cluster. Starting replication creates a follower index from scratch; you can't convert an existing index to a follower index.
|
||||
|
||||
## Confirm replication
|
||||
|
||||
After replication starts, get the status:
|
||||
|
||||
```bash
|
||||
curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_status?pretty'
|
||||
|
||||
{
|
||||
"status" : "SYNCING",
|
||||
"reason" : "User initiated",
|
||||
"leader_alias" : "my-connection-alias",
|
||||
"leader_index" : "leader-01",
|
||||
"follower_index" : "follower-01",
|
||||
"syncing_details" : {
|
||||
"leader_checkpoint" : -1,
|
||||
"follower_checkpoint" : -1,
|
||||
"seq_no" : 0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Possible statuses are `SYNCING`, `BOOTSTRAPING`, `PAUSED`, and `REPLICATION NOT IN PROGRESS`.
|
||||
|
||||
The leader and follower checkpoint values begin as negative numbers and reflect the shard count (-1 for one shard, -5 for five shards, and so on). The values increment with each change and illustrate how many updates the follower is behind the leader. If the indices are fully synced, the values are the same.
|
||||
|
||||
To confirm that replication is actually happening, add a document to the leader index:
|
||||
|
||||
```bash
|
||||
curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/leader-01/_doc/1?pretty' -d '{"The Shining": "Stephen King"}'
|
||||
```
|
||||
|
||||
Then validate the replicated content on the follower index:
|
||||
|
||||
```bash
|
||||
curl -XGET -k -u 'admin:admin' 'https://localhost:9200/follower-01/_search?pretty'
|
||||
|
||||
{
|
||||
...
|
||||
"hits": [{
|
||||
"_index": "follower-01",
|
||||
"_type": "_doc",
|
||||
"_id": "1",
|
||||
"_score": 1.0,
|
||||
"_source": {
|
||||
"The Shining": "Stephen King"
|
||||
}
|
||||
}]
|
||||
}
|
||||
```
|
||||
|
||||
## Pause and resume replication
|
||||
|
||||
You can temporarily pause replication of an index if you need to remediate issues or reduce load on the leader cluster:
|
||||
|
||||
```bash
|
||||
curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_pause?pretty' -d '{}'
|
||||
```
|
||||
|
||||
To confirm replication is paused, get the status:
|
||||
|
||||
```bash
|
||||
curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_status?pretty'
|
||||
|
||||
{
|
||||
"status" : "PAUSED",
|
||||
"reason" : "User initiated",
|
||||
"leader_alias" : "my-connection-alias",
|
||||
"leader_index" : "leader-01",
|
||||
"follower_index" : "follower-01"
|
||||
}
|
||||
```
|
||||
|
||||
When you're done making changes, resume replication:
|
||||
|
||||
```bash
|
||||
curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_resume?pretty' -d '{}'
|
||||
```
|
||||
|
||||
When replication resumes, the follower index picks up any changes that were made to the leader index while replication was paused.
|
||||
|
||||
Note that you can't resume replication after it's been paused for more than 12 hours. You must [stop replication]({{site.url}}{{site.baseurl}}/replication-plugin/api/#stop-replication), delete the follower index, and restart replication of the leader.
|
||||
|
||||
## Stop replication
|
||||
|
||||
Terminate replication of a specified index from the follower cluster:
|
||||
|
||||
```bash
|
||||
curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_stop?pretty' -d '{}'
|
||||
```
|
||||
|
||||
When you stop replication, the follower index un-follows the leader and becomes a standard index that you can write to. You can't restart replication after stopping it.
|
||||
|
||||
Get the status to confirm that the index is no longer being replicated:
|
||||
|
||||
```bash
|
||||
curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_status?pretty'
|
||||
|
||||
{
|
||||
"status" : "REPLICATION NOT IN PROGRESS"
|
||||
}
|
||||
```
|
||||
|
||||
You can further confirm that replication is stopped by making modifications to the leader index and confirming they don't show up on the follower index.
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
---
|
||||
layout: default
|
||||
title: Cross-cluster replication
|
||||
nav_order: 1
|
||||
has_children: false
|
||||
redirect_from:
|
||||
- /replication-plugin/
|
||||
---
|
||||
|
||||
# Cross-cluster replication
|
||||
|
||||
The cross-cluster replication plugin lets you replicate indices, mappings, and metadata from one OpenSearch cluster to another. Cross-cluster replication has the following benefits:
|
||||
- By replicating your indices, you ensure that you can continue to handle search requests in the event of an outage.
|
||||
- Replicating data across geographically distant data centers minimizes the distance between the data and the application server, reducing expensive latencies.
|
||||
- You can replicate data from multiple smaller clusters to a centralized reporting cluster, which is useful when it's inefficient to query across a large network.
|
||||
|
||||
Replication follows an active-passive model where the follower index (where the data is replicated) pulls data from the leader (remote) index.
|
||||
|
||||
The replication plugin supports replication of indices using wildcard pattern matching and provides commands to pause, resume, and stop replication. Once replication starts on an index, it initiates persistent background tasks on all primary shards on the follower cluster, which continuously poll corresponding shards from the leader cluster for updates.
|
||||
|
||||
You can use the replication plugin with the security plugin to encrypt cross-cluster traffic with node-to-node encryption and control access to replication activities.
|
||||
|
||||
To start, see [Get started with cross-cluster replication]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/).
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
---
|
||||
layout: default
|
||||
title: Permissions
|
||||
nav_order: 30
|
||||
---
|
||||
|
||||
# Cross-cluster replication permissions
|
||||
|
||||
You can use the [security plugin]({{site.url}}{{site.baseurl}}/security-plugin/index/) with cross-cluster replication to limit users to certain actions. For example, you might want certain users to only perform replication activity on the leader or follower cluster.
|
||||
|
||||
Because cross-cluster replication involves multiple clusters, it's possible that clusters might have different security configurations. The following configurations are supported:
|
||||
|
||||
- Security plugin fully enabled on both clusters
|
||||
- Security plugin enabled only for TLS on both clusters (`plugins.security.ssl_only`)
|
||||
- Security plugin absent or disabled on both clusters (not recommended)
|
||||
|
||||
Enable node-to-node encryption on both the leader and the follower cluster to ensure that replication traffic between the clusters is encrypted.
|
||||
|
||||
## Basic permissions
|
||||
|
||||
In order for non-admin users to perform replication activities, they must be mapped to the appropriate permissions.
|
||||
|
||||
The security plugin has two built-in roles that cover most replication use cases: `cross_cluster_replication_leader_full_access`, which provides replication permissions on the leader cluster, and `cross_cluster_replication_follower_full_access`, which provides replication permissions on the follower cluster. For descriptions of each, see [Predefined roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/users-roles#predefined-roles).
|
||||
|
||||
If you don't want to use the default roles, you can combine individual replication [permissions]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#replication-permissions) to meet your needs. Most permissions correspond to specific REST API operations. For example, the `indices:admin/plugins/replication/index/pause` permission lets you pause replication.
|
||||
|
||||
## Map the leader and follower cluster roles
|
||||
|
||||
The [start replication]({{site.url}}{{site.baseurl}}/replication-plugin/api/#start-replication) and [create replication rule]({{site.url}}{{site.baseurl}}/replication-plugin/api/#create-replication-rule) operations are special cases. They involve background processes on the leader and follower clusters that must be associated with roles. When you perform one of these actions, you must explicitly pass the `leader_cluster_role` and
|
||||
`follower_cluster_role` in the request, which OpenSearch then uses in all backend replication tasks.
|
||||
|
||||
To enable non-admins to start replication and create replication rules, create an identical user on each cluster (for example, `replication_user`) and map them to the `cross_cluster_replication_leader_full_access` role on the remote cluster and `cross_cluster_replication_follower_full_access` on the follower cluster. For instructions, see [Map users to roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/users-roles/#map-users-to-roles).
|
||||
|
||||
Then add those roles to the request, and sign it with the appropriate credentials:
|
||||
|
||||
```bash
|
||||
curl -XPUT -k -H 'Content-Type: application/json' -u 'replication_user:password' 'https://localhost:9200/_plugins/_replication/follower-01/_start?pretty' -d '
|
||||
{
|
||||
"leader_alias": "leader-cluster",
|
||||
"leader_index": "leader-01",
|
||||
"use_roles":{
|
||||
"leader_cluster_role": "cross_cluster_replication_leader_full_access",
|
||||
"follower_cluster_role": "cross_cluster_replication_follower_full_access"
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
You can create your own, custom leader and follower cluster roles using individual permissions, but we recommend using the default roles, which are a good fit for most use cases.
|
||||
|
||||
## Replication permissions
|
||||
|
||||
The following sections list the available index and cluster-level permissions for cross-cluster replication.
|
||||
|
||||
### Follower cluster
|
||||
|
||||
The security plugin supports these permissions for the follower cluster:
|
||||
|
||||
```
|
||||
indices:admin/plugins/replication/index/setup/validate
|
||||
indices:admin/plugins/replication/index/start
|
||||
indices:admin/plugins/replication/index/pause
|
||||
indices:admin/plugins/replication/index/resume
|
||||
indices:admin/plugins/replication/index/stop
|
||||
indices:admin/plugins/replication/index/update
|
||||
indices:admin/plugins/replication/index/status_check
|
||||
indices:data/write/plugins/replication/changes
|
||||
cluster:admin/plugins/replication/autofollow/update
|
||||
```
|
||||
|
||||
### Leader cluster
|
||||
|
||||
The security plugin supports these permissions for the leader cluster:
|
||||
|
||||
```
|
||||
indices:admin/plugins/replication/validate
|
||||
indices:data/read/plugins/replication/file_chunk
|
||||
indices:data/read/plugins/replication/changes
|
||||
```
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
---
|
||||
layout: default
|
||||
title: Settings
|
||||
nav_order: 40
|
||||
---
|
||||
|
||||
# Replication settings
|
||||
|
||||
The replication plugin adds several settings to the standard OpenSearch cluster settings.
|
||||
The settings are dynamic, so you can change the default behavior of the plugin without restarting your cluster.
|
||||
You can mark settings as `persistent` or `transient`.
|
||||
|
||||
For example, to update how often the follower cluster polls the leader cluster for updates:
|
||||
|
||||
```json
|
||||
PUT _cluster/settings
|
||||
{
|
||||
"persistent": {
|
||||
"plugins.replication.follower.metadata_sync_interval": "30s"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
These settings manage the resources consumed by remote recoveries. We don’t recommend changing these settings; the defaults should work well for most use cases.
|
||||
|
||||
Setting | Default | Description
|
||||
:--- | :--- | :---
|
||||
`plugins.replication.follower.index.recovery.chunk_size` | 10MB | The chunk size requested by the follower cluster during file transfer. Specify the chunk size as a value and unit, for example, 10MB, 5KB. See [Supported units]({{site.url}}{{site.baseurl}}/opensearch/units/).
|
||||
`plugins.replication.follower.index.recovery.max_concurrent_file_chunks` | 4 | The number of file chunk requests that can be sent in parallel for each recovery.
|
||||
`plugins.replication.follower.index.ops_batch_size` | 5000 | The number of operations that can be fetched at a time during the syncing phase of replication.
|
||||
`plugins.replication.follower.concurrent_readers_per_shard` | 2 | The number of concurrent requests from the follower cluster per shard during the syncing phase of replication.
|
||||
`plugins.replication.autofollow.fetch_poll_interval` | 30s | How often auto-follow tasks poll the leader cluster for new matching indices.
|
||||
`plugins.replication.follower.metadata_sync_interval` | 60s | How often the follower cluster polls the leader cluster for updated index metadata.
|
||||
`plugins.replication.translog.retention_lease.pruning.enabled` | true | If enabled, prunes the translog based on retention leases on the leader index.
|
||||
`plugins.replication.translog.retention_size` | 512MB | Controls the size of the translog on the leader index.
|
||||
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
$content-width: 740px;
|
||||
|
||||
//
|
||||
// Brand colors
|
||||
//
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
$content-width: 740px !default;
|
||||
|
||||
@import "../../assets/fonts/open-sans/webfont.css";
|
||||
@import "../../assets/fonts/open-sans-condensed/webfont.css";
|
||||
@import "../../assets/fonts/noto-serif/webfont.css";
|
||||
|
|
@ -54,13 +56,13 @@ body {
|
|||
|
||||
code {
|
||||
@include monospace;
|
||||
font-size: 0.75rem;
|
||||
font-size: 0.8rem;
|
||||
-webkit-font-smoothing: subpixel-antialiased;
|
||||
-moz-osx-font-smoothing: auto;
|
||||
}
|
||||
|
||||
.site-nav {
|
||||
padding-top: 2rem;
|
||||
padding-top: 1rem;
|
||||
}
|
||||
|
||||
.nav-category {
|
||||
|
|
@ -108,6 +110,13 @@ code {
|
|||
}
|
||||
}
|
||||
|
||||
.site-footer {
|
||||
display: none;
|
||||
@include mq(lg) {
|
||||
display: block;
|
||||
}
|
||||
}
|
||||
|
||||
.external-arrow {
|
||||
position: relative;
|
||||
top: 0.125rem;
|
||||
|
|
@ -389,7 +398,7 @@ html {
|
|||
|
||||
body {
|
||||
@include serif;
|
||||
@include font-size(18);
|
||||
@include font-size(16);
|
||||
background: $background-lightest;
|
||||
color: $text;
|
||||
line-height: 1.6;
|
||||
|
|
@ -914,7 +923,7 @@ main {
|
|||
@include clearfix;
|
||||
}
|
||||
|
||||
@media screen and (max-width: 799px) {
|
||||
@media screen and (max-width: 46.24rem) {
|
||||
[role="banner"] .nav-menu-on .small-nav {
|
||||
|
||||
li {
|
||||
|
|
@ -962,6 +971,7 @@ main {
|
|||
line-height: 1.3;
|
||||
padding: 1px 0 6px;
|
||||
margin: .45em 0 .35em;
|
||||
letter-spacing: -1px;
|
||||
|
||||
@include mq(md) {
|
||||
@include font-size(32, true);
|
||||
|
|
@ -975,6 +985,7 @@ main {
|
|||
a {
|
||||
font-weight: 300;
|
||||
background: none;
|
||||
color: $text-link-alternate;
|
||||
|
||||
&:hover, :active {
|
||||
background: none;
|
||||
|
|
@ -1082,6 +1093,11 @@ main {
|
|||
@include sans-serif;
|
||||
@include warning-stripes;
|
||||
|
||||
/* To match the website */
|
||||
font-size: 1.125rem;
|
||||
text-decoration-thickness: 0.5px;
|
||||
text-underline-offset: 1px;
|
||||
|
||||
a {
|
||||
color: $text;
|
||||
|
||||
|
|
@ -1102,3 +1118,119 @@ main {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
.version-wrapper {
|
||||
text-align: center;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
version-selector {
|
||||
z-index: 1;
|
||||
font-size: .9rem;
|
||||
|
||||
--normal-bg: linear-gradient(#{lighten($blue-300, 5%)}, #{darken($blue-300, 2%)});
|
||||
--hover-bg: linear-gradient(#{lighten($blue-300, 2%)}, #{darken($blue-300, 4%)});
|
||||
--link-color: #{$blue-300};
|
||||
}
|
||||
|
||||
.custom-search-results {
|
||||
& > div {
|
||||
padding: 1rem;
|
||||
}
|
||||
|
||||
cite {
|
||||
@include font-size(12);
|
||||
@include sans-serif;
|
||||
color: $grey-dk-300;
|
||||
text-decoration: none;
|
||||
font-style: normal;
|
||||
display: block;
|
||||
line-height: 1;
|
||||
font-weight: normal;
|
||||
}
|
||||
|
||||
a {
|
||||
@include font-size(20);
|
||||
@include heading-sans-serif;
|
||||
line-height: 1.6;
|
||||
font-weight: bold;
|
||||
outline: none;
|
||||
}
|
||||
|
||||
span {
|
||||
@include font-size(14);
|
||||
color: $grey-dk-200;
|
||||
line-height: 1.4;
|
||||
display: block;
|
||||
overflow-wrap: break-word;
|
||||
|
||||
&:only-child {
|
||||
text-align: center;
|
||||
padding: 1rem;
|
||||
}
|
||||
}
|
||||
|
||||
.highlighted {
|
||||
background: #EAF4F9;
|
||||
}
|
||||
}
|
||||
|
||||
.banner-alert ~ main .custom-search-results {
|
||||
max-height: calc(100vh - 200% - 60px - 3.6rem) !important;
|
||||
}
|
||||
|
||||
.search-spinner {
|
||||
display: none;
|
||||
font-weight: 700;
|
||||
outline: 0;
|
||||
user-select: none;
|
||||
|
||||
position: absolute;
|
||||
padding-left: 0.6rem;
|
||||
height: 100%;
|
||||
|
||||
&.spinning {
|
||||
display: flex;
|
||||
|
||||
& ~ .search-label {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.search-spinner > i {
|
||||
border-color: rgba($grey-dk-000, 0.2);
|
||||
position: relative;
|
||||
animation: spin 0.6s infinite linear;
|
||||
border-width: 3px;
|
||||
border-style: solid;
|
||||
border-radius: 100%;
|
||||
display: inline-block;
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
vertical-align: middle;
|
||||
align-self: center;
|
||||
|
||||
&:before {
|
||||
content: "";
|
||||
border: 3px solid rgba($grey-dk-000, 0);
|
||||
border-top-color: rgba($grey-dk-000, 0.8);
|
||||
border-radius: 100%;
|
||||
display: block;
|
||||
left: -3px;
|
||||
position: absolute;
|
||||
top: -3px;
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
box-sizing: content-box;
|
||||
}
|
||||
}
|
||||
|
||||
@keyframes spin {
|
||||
from {
|
||||
transform: rotate(0deg)
|
||||
}
|
||||
to {
|
||||
transform: rotate(359deg)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
---
|
||||
layout: default
|
||||
title: k-NN API
|
||||
title: API
|
||||
nav_order: 5
|
||||
parent: k-NN
|
||||
has_children: false
|
||||
|
|
@ -8,7 +8,7 @@ has_children: false
|
|||
|
||||
# k-NN plugin API
|
||||
|
||||
The k-NN plugin adds two API operations to help you better manage the plugin's functionality.
|
||||
The k-NN plugin adds several APIs for managing, monitoring and optimizing your k-NN workload.
|
||||
|
||||
|
||||
## Stats
|
||||
|
|
@ -23,25 +23,35 @@ GET /_plugins/_knn/nodeId1,nodeId2/stats/statName1,statName2
|
|||
Statistic | Description
|
||||
:--- | :---
|
||||
`circuit_breaker_triggered` | Indicates whether the circuit breaker is triggered. This statistic is only relevant to approximate k-NN search.
|
||||
`total_load_time` | The time in nanoseconds that k-NN has taken to load graphs into the cache. This statistic is only relevant to approximate k-NN search.
|
||||
`eviction_count` | The number of graphs that have been evicted from the cache due to memory constraints or idle time. This statistic is only relevant to approximate k-NN search. <br /> **Note**: Explicit evictions that occur because of index deletion aren't counted.
|
||||
`hit_count` | The number of cache hits. A cache hit occurs when a user queries a graph that's already loaded into memory. This statistic is only relevant to approximate k-NN search.
|
||||
`miss_count` | The number of cache misses. A cache miss occurs when a user queries a graph that isn't loaded into memory yet. This statistic is only relevant to approximate k-NN search.
|
||||
`graph_memory_usage` | Current cache size (total size of all graphs in memory) in kilobytes. This statistic is only relevant to approximate k-NN search.
|
||||
`graph_memory_usage_percentage` | The current weight of the cache as a percentage of the maximum cache capacity.
|
||||
`graph_index_requests` | The number of requests to add the `knn_vector` field of a document into a graph.
|
||||
`graph_index_errors` | The number of requests to add the `knn_vector` field of a document into a graph that have produced an error.
|
||||
`graph_query_requests` | The number of graph queries that have been made.
|
||||
`graph_query_errors` | The number of graph queries that have produced an error.
|
||||
`total_load_time` | The time in nanoseconds that k-NN has taken to load native library indices into the cache. This statistic is only relevant to approximate k-NN search.
|
||||
`eviction_count` | The number of native library indices that have been evicted from the cache due to memory constraints or idle time. This statistic is only relevant to approximate k-NN search. <br /> **Note**: Explicit evictions that occur because of index deletion aren't counted.
|
||||
`hit_count` | The number of cache hits. A cache hit occurs when a user queries a native library index that's already loaded into memory. This statistic is only relevant to approximate k-NN search.
|
||||
`miss_count` | The number of cache misses. A cache miss occurs when a user queries a native library index that isn't loaded into memory yet. This statistic is only relevant to approximate k-NN search.
|
||||
`graph_memory_usage` | The amount of native memory native library indices are using on the node in kilobytes.
|
||||
`graph_memory_usage_percentage` | The amount of native memory native library indices are using on the node as a percentage of the maximum cache capacity.
|
||||
`graph_index_requests` | The number of requests to add the `knn_vector` field of a document into a native library index.
|
||||
`graph_index_errors` | The number of requests to add the `knn_vector` field of a document into a native library index that have produced an error.
|
||||
`graph_query_requests` | The number of native library index queries that have been made.
|
||||
`graph_query_errors` | The number of native library index queries that have produced an error.
|
||||
`knn_query_requests` | The number of k-NN query requests received.
|
||||
`cache_capacity_reached` | Whether `knn.memory.circuit_breaker.limit` has been reached. This statistic is only relevant to approximate k-NN search.
|
||||
`load_success_count` | The number of times k-NN successfully loaded a graph into the cache. This statistic is only relevant to approximate k-NN search.
|
||||
`load_exception_count` | The number of times an exception occurred when trying to load a graph into the cache. This statistic is only relevant to approximate k-NN search.
|
||||
`indices_in_cache` | For each index that has graphs in the cache, this statistic provides the number of graphs that index has and the total `graph_memory_usage` that index is using, in kilobytes.
|
||||
`load_success_count` | The number of times k-NN successfully loaded a native library index into the cache. This statistic is only relevant to approximate k-NN search.
|
||||
`load_exception_count` | The number of times an exception occurred when trying to load a native library index into the cache. This statistic is only relevant to approximate k-NN search.
|
||||
`indices_in_cache` | For each OpenSearch index with a `knn_vector` field and approximate k-NN turned on, this statistic provides the number of native library indices that OpenSearch index has and the total `graph_memory_usage` that the OpenSearch index is using, in kilobytes.
|
||||
`script_compilations` | The number of times the k-NN script has been compiled. This value should usually be 1 or 0, but if the cache containing the compiled scripts is filled, the k-NN script might be recompiled. This statistic is only relevant to k-NN score script search.
|
||||
`script_compilation_errors` | The number of errors during script compilation. This statistic is only relevant to k-NN score script search.
|
||||
`script_query_requests` | The total number of script queries. This statistic is only relevant to k-NN score script search.
|
||||
`script_query_errors` | The number of errors during script queries. This statistic is only relevant to k-NN score script search.
|
||||
`nmslib_initialized` | Boolean value indicating whether the *nmslib* JNI library has been loaded and initialized on the node.
|
||||
`faiss_initialized` | Boolean value indicating whether the *faiss* JNI library has been loaded and initialized on the node.
|
||||
`model_index_status` | Status of model system index. Valid values are "red", "yellow", "green". If the index does not exist, this will be null.
|
||||
`indexing_from_model_degraded` | Boolean value indicating if indexing from a model is degraded. This will happen if there is not enough JVM memory to cache the models.
|
||||
`training_requests` | The number of training requests made to the node.
|
||||
`training_errors` | The number of training errors that have occurred on the node.
|
||||
`training_memory_usage` | The amount of native memory training is using on the node in kilobytes.
|
||||
`training_memory_usage_percentage` | The amount of native memory training is using on the node as a percentage of the maximum cache capacity.
|
||||
|
||||
**Note**: Some stats contain *graph* in the name. In these cases, *graph* is synonymous with *native library index*. The term *graph* is a legacy detail, coming from when the plugin only supported the HNSW algorithm, which consists of hierarchical graphs.
|
||||
|
||||
|
||||
### Usage
|
||||
|
|
@ -54,37 +64,45 @@ GET /_plugins/_knn/stats?pretty
|
|||
"successful" : 1,
|
||||
"failed" : 0
|
||||
},
|
||||
"cluster_name" : "_run",
|
||||
"cluster_name" : "my-cluster",
|
||||
"circuit_breaker_triggered" : false,
|
||||
"model_index_status" : "YELLOW",
|
||||
"nodes" : {
|
||||
"HYMrXXsBSamUkcAjhjeN0w" : {
|
||||
"eviction_count" : 0,
|
||||
"miss_count" : 1,
|
||||
"graph_memory_usage" : 1,
|
||||
"graph_memory_usage_percentage" : 3.68,
|
||||
"graph_index_requests" : 7,
|
||||
"graph_index_errors" : 1,
|
||||
"knn_query_requests" : 4,
|
||||
"graph_query_requests" : 30,
|
||||
"graph_query_errors" : 15,
|
||||
"indices_in_cache" : {
|
||||
"myindex" : {
|
||||
"graph_memory_usage" : 2,
|
||||
"graph_memory_usage_percentage" : 3.68,
|
||||
"graph_count" : 2
|
||||
}
|
||||
},
|
||||
"cache_capacity_reached" : false,
|
||||
"load_exception_count" : 0,
|
||||
"hit_count" : 0,
|
||||
"load_success_count" : 1,
|
||||
"total_load_time" : 2878745,
|
||||
"script_compilations" : 1,
|
||||
"script_compilation_errors" : 0,
|
||||
"script_query_requests" : 534,
|
||||
"script_query_errors" : 0
|
||||
}
|
||||
"JdfxIkOS1-43UxqNz98nw" : {
|
||||
"graph_memory_usage_percentage" : 3.68,
|
||||
"graph_query_requests" : 1420920,
|
||||
"graph_memory_usage" : 2,
|
||||
"cache_capacity_reached" : false,
|
||||
"load_success_count" : 179,
|
||||
"training_memory_usage" : 0,
|
||||
"indices_in_cache" : {
|
||||
"myindex" : {
|
||||
"graph_memory_usage" : 2,
|
||||
"graph_memory_usage_percentage" : 3.68,
|
||||
"graph_count" : 2
|
||||
}
|
||||
},
|
||||
"script_query_errors" : 0,
|
||||
"hit_count" : 1420775,
|
||||
"knn_query_requests" : 147092,
|
||||
"total_load_time" : 2436679306,
|
||||
"miss_count" : 179,
|
||||
"training_memory_usage_percentage" : 0.0,
|
||||
"graph_index_requests" : 656,
|
||||
"faiss_initialized" : true,
|
||||
"load_exception_count" : 0,
|
||||
"training_errors" : 0,
|
||||
"eviction_count" : 0,
|
||||
"nmslib_initialized" : false,
|
||||
"script_compilations" : 0,
|
||||
"script_query_requests" : 0,
|
||||
"graph_query_errors" : 0,
|
||||
"indexing_from_model_degraded" : false,
|
||||
"graph_index_errors" : 0,
|
||||
"training_requests" : 17,
|
||||
"script_compilation_errors" : 0
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
|
@ -96,7 +114,7 @@ GET /_plugins/_knn/HYMrXXsBSamUkcAjhjeN0w/stats/circuit_breaker_triggered,graph_
|
|||
"successful" : 1,
|
||||
"failed" : 0
|
||||
},
|
||||
"cluster_name" : "_run",
|
||||
"cluster_name" : "my-cluster",
|
||||
"circuit_breaker_triggered" : false,
|
||||
"nodes" : {
|
||||
"HYMrXXsBSamUkcAjhjeN0w" : {
|
||||
|
|
@ -111,13 +129,13 @@ GET /_plugins/_knn/HYMrXXsBSamUkcAjhjeN0w/stats/circuit_breaker_triggered,graph_
|
|||
Introduced 1.0
|
||||
{: .label .label-purple }
|
||||
|
||||
The Hierarchical Navigable Small World (HNSW) graphs used to perform an approximate k-Nearest Neighbor (k-NN) search are stored as `.hnsw` files with other Apache Lucene segment files. In order for you to perform a search on these graphs using the k-NN plugin, the plugin needs to load these files into native memory.
|
||||
The native library indices used to perform approximate k-Nearest Neighbor (k-NN) search are stored as special files with other Apache Lucene segment files. In order for you to perform a search on these indices using the k-NN plugin, the plugin needs to load these files into native memory.
|
||||
|
||||
If the plugin hasn't loaded the graphs into native memory, it loads them when it receives a search request. The loading time can cause high latency during initial queries. To avoid this situation, users often run random queries during a warmup period. After this warmup period, the graphs are loaded into native memory and their production workloads can begin. This loading process is indirect and requires extra effort.
|
||||
If the plugin hasn't loaded the files into native memory, it loads them when it receives a search request. The loading time can cause high latency during initial queries. To avoid this situation, users often run random queries during a warmup period. After this warmup period, the files are loaded into native memory and their production workloads can begin. This loading process is indirect and requires extra effort.
|
||||
|
||||
As an alternative, you can avoid this latency issue by running the k-NN plugin warmup API operation on whatever indices you're interested in searching. This operation loads all the graphs for all of the shards (primaries and replicas) of all the indices specified in the request into native memory.
|
||||
As an alternative, you can avoid this latency issue by running the k-NN plugin warmup API operation on whatever indices you're interested in searching. This operation loads all the native library files for all of the shards (primaries and replicas) of all the indices specified in the request into native memory.
|
||||
|
||||
After the process finishes, you can start searching against the indices with no initial latency penalties. The warmup API operation is idempotent, so if a segment's graphs are already loaded into memory, this operation has no impact on those graphs. It only loads graphs that aren't currently in memory.
|
||||
After the process finishes, you can start searching against the indices with no initial latency penalties. The warmup API operation is idempotent, so if a segment's native library files are already loaded into memory, this operation has no impact. It only loads files that aren't currently in memory.
|
||||
|
||||
|
||||
### Usage
|
||||
|
|
@ -150,8 +168,212 @@ After the operation has finished, use the [k-NN `_stats` API operation](#stats)
|
|||
|
||||
For the warmup operation to function properly, follow these best practices:
|
||||
|
||||
* Don't run merge operations on indices that you want to warm up. During merge, the k-NN plugin creates new segments, and old segments are sometimes deleted. For example, you could encounter a situation in which the warmup API operation loads graphs A and B into native memory, but segment C is created from segments A and B being merged. The graphs for A and B would no longer be in memory, and graph C would also not be in memory. In this case, the initial penalty for loading graph C is still present.
|
||||
* Don't run merge operations on indices that you want to warm up. During merge, the k-NN plugin creates new segments, and old segments are sometimes deleted. For example, you could encounter a situation in which the warmup API operation loads native library indices A and B into native memory, but segment C is created from segments A and B being merged. The native library indices for A and B would no longer be in memory, and native library index C would also not be in memory. In this case, the initial penalty for loading native library index C is still present.
|
||||
|
||||
* Confirm that all graphs you want to warm up can fit into native memory. For more information about the native memory limit, see the [knn.memory.circuit_breaker.limit statistic]({{site.url}}{{site.baseurl}}/search-plugins/knn/settings#cluster-settings). High graph memory usage causes cache thrashing, which can lead to operations constantly failing and attempting to run again.
|
||||
* Confirm that all native library indices you want to warm up can fit into native memory. For more information about the native memory limit, see the [knn.memory.circuit_breaker.limit statistic]({{site.url}}{{site.baseurl}}/search-plugins/knn/settings#cluster-settings). High graph memory usage causes cache thrashing, which can lead to operations constantly failing and attempting to run again.
|
||||
|
||||
* Don't index any documents that you want to load into the cache. Writing new information to segments prevents the warmup API operation from loading the graphs until they're searchable. This means that you would have to run the warmup operation again after indexing finishes.
|
||||
* Don't index any documents that you want to load into the cache. Writing new information to segments prevents the warmup API operation from loading the native library indices until they're searchable. This means that you would have to run the warmup operation again after indexing finishes.
|
||||
|
||||
## Get Model
|
||||
Introduced 1.2
|
||||
{: .label .label-purple }
|
||||
|
||||
Used to retrieve information about models present in the cluster. Some native library index configurations require a
|
||||
training step before indexing and querying can begin. The output of training is a model that can then be used to
|
||||
initialize native library index files during indexing. The model is serialized in the k-NN model system index.
|
||||
|
||||
```
|
||||
GET /_plugins/_knn/models/{model_id}
|
||||
```
|
||||
|
||||
Response Field | Description
|
||||
:--- | :---
|
||||
`model_id` | The id of the fetched model.
|
||||
`model_blob` | The base64 encoded string of the serialized model.
|
||||
`state` | Current state of the model. Either "created", "failed", "training".
|
||||
`timestamp` | Time when the model was created.
|
||||
`description` | User provided description of the model.
|
||||
`error` | Error message explaining why the model is in the failed state.
|
||||
`space_type` | Space type this model is trained for.
|
||||
`dimension` | Dimension this model is for.
|
||||
`engine` | Native library used to create model. Either "faiss" or "nmslib".
|
||||
|
||||
### Usage
|
||||
|
||||
```json
|
||||
GET /_plugins/_knn/models/test-model?pretty
|
||||
{
|
||||
"model_id" : "test-model",
|
||||
"model_blob" : "SXdGbIAAAAAAAAAAAA...",
|
||||
"state" : "created",
|
||||
"timestamp" : "2021-11-15T18:45:07.505369036Z",
|
||||
"description" : "Default",
|
||||
"error" : "",
|
||||
"space_type" : "l2",
|
||||
"dimension" : 128,
|
||||
"engine" : "faiss"
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
GET /_plugins/_knn/models/test-model?pretty&filter_path=model_id,state
|
||||
{
|
||||
"model_id" : "test-model",
|
||||
"state" : "created"
|
||||
}
|
||||
```
|
||||
|
||||
## Search Model
|
||||
Introduced 1.2
|
||||
{: .label .label-purple }
|
||||
|
||||
Use an OpenSearch query to search for models in the index.
|
||||
|
||||
### Usage
|
||||
```json
|
||||
GET/POST /_plugins/_knn/models/_search?pretty&_source_excludes=model_blob
|
||||
{
|
||||
"query": {
|
||||
...
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
"took" : 0,
|
||||
"timed_out" : false,
|
||||
"_shards" : {
|
||||
"total" : 1,
|
||||
"successful" : 1,
|
||||
"skipped" : 0,
|
||||
"failed" : 0
|
||||
},
|
||||
"hits" : {
|
||||
"total" : {
|
||||
"value" : 1,
|
||||
"relation" : "eq"
|
||||
},
|
||||
"max_score" : 1.0,
|
||||
"hits" : [
|
||||
{
|
||||
"_index" : ".opensearch-knn-models",
|
||||
"_type" : "_doc",
|
||||
"_id" : "test-model",
|
||||
"_score" : 1.0,
|
||||
"_source" : {
|
||||
"engine" : "faiss",
|
||||
"space_type" : "l2",
|
||||
"description" : "Default",
|
||||
"model_id" : "test-model",
|
||||
"state" : "created",
|
||||
"error" : "",
|
||||
"dimension" : 128,
|
||||
"timestamp" : "2021-11-15T18:45:07.505369036Z"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Delete Model
|
||||
Introduced 1.2
|
||||
{: .label .label-purple }
|
||||
|
||||
Used to delete a particular model in the cluster.
|
||||
|
||||
### Usage
|
||||
|
||||
```json
|
||||
DELETE /_plugins/_knn/models/{model_id}
|
||||
{
|
||||
"model_id": {model_id},
|
||||
"acknowledged": true
|
||||
}
|
||||
```
|
||||
|
||||
## Train Model
|
||||
Introduced 1.2
|
||||
{: .label .label-purple }
|
||||
|
||||
Create and train a model that can be used for initializing k-NN native library indices during indexing. This API will
|
||||
pull training data from a `knn_vector` field in a training index and then create and train a model and then serialize it
|
||||
to the model system index. Training data must match the dimension passed into the body of the request. This request
|
||||
will return when training begins. To monitor the state of the model, use the [Get model API](#get-model).
|
||||
|
||||
Query Parameter | Description
|
||||
:--- | :---
|
||||
`model_id` | (Optional) The id of the fetched model. If not specified, a random id will be generated.
|
||||
`node_id` | (Optional) Preferred node to execute training. If set, this node will be used to perform training if it is deemed to be capable.
|
||||
|
||||
Request Parameter | Description
|
||||
:--- | :---
|
||||
`training_index` | Index from where training data from.
|
||||
`training_field` | `knn_vector` field from `training_index` to grab training data from. Dimension of this field must match `dimension` passed in to this request.
|
||||
`dimension` | Dimension this model is for.
|
||||
`max_training_vector_count` | (Optional) Maximum number of vectors from the training index to use for training. Defaults to all of the vectors in the index.
|
||||
`search_size` | (Optional) Training data is pulled from the training index with scroll queries. Defines the number of results to return per scroll query. Defaults to 10,000.
|
||||
`description` | (Optional) User provided description of the model.
|
||||
`method` | Configuration of ANN method used for search. For more information on possible methods, refer to the [method documentation]({{site.url}}{{site.baseurl}}/search-plugins/knn/knn-index#method-definitions). Method must require training to be valid.
|
||||
|
||||
|
||||
### Usage
|
||||
|
||||
```json
|
||||
POST /_plugins/_knn/models/{model_id}/_train?preference={node_id}
|
||||
{
|
||||
"training_index": "train-index-name",
|
||||
"training_field": "train-field-name",
|
||||
"dimension": 16,
|
||||
"max_training_vector_count": 1200,
|
||||
"search_size": 100,
|
||||
"description": "My model",
|
||||
"method": {
|
||||
"name":"ivf",
|
||||
"engine":"faiss",
|
||||
"space_type": "l2",
|
||||
"parameters":{
|
||||
"nlists":128,
|
||||
"encoder":{
|
||||
"name":"pq",
|
||||
"parameters":{
|
||||
"code_size":8
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
"model_id": "model_x"
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
POST /_plugins/_knn/models/_train?preference={node_id}
|
||||
{
|
||||
"training_index": "train-index-name",
|
||||
"training_field": "train-field-name",
|
||||
"dimension": 16,
|
||||
"max_training_vector_count": 1200,
|
||||
"search_size": 100,
|
||||
"description": "My model",
|
||||
"method": {
|
||||
"name":"ivf",
|
||||
"engine":"faiss",
|
||||
"space_type": "l2",
|
||||
"parameters":{
|
||||
"nlists":128,
|
||||
"encoder":{
|
||||
"name":"pq",
|
||||
"parameters":{
|
||||
"code_size":8
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
"model_id": "dcdwscddscsad"
|
||||
}
|
||||
```
|
||||
|
|
|
|||
|
|
@ -9,21 +9,28 @@ has_math: true
|
|||
|
||||
# Approximate k-NN search
|
||||
|
||||
The approximate k-NN method uses [nmslib's](https://github.com/nmslib/nmslib/) implementation of the Hierarchical Navigable Small World (HNSW) algorithm to power k-NN search. In this case, approximate means that for a given search, the neighbors returned are an estimate of the true k-nearest neighbors. Of the three methods, this method offers the best search scalability for large data sets. Generally speaking, once the data set gets into the hundreds of thousands of vectors, this approach is preferred.
|
||||
The approximate k-NN search method uses nearest neighbor algorithms from *nmslib* and *faiss* to power
|
||||
k-NN search. To see the algorithms that the plugin currently supports, check out the [k-NN Index documentation]({{site.url}}{{site.baseurl}}/search-plugins/knn/knn-index#method-definitions).
|
||||
In this case, approximate means that for a given search, the neighbors returned are an estimate of the true k-nearest
|
||||
neighbors. Of the three search methods the plugin provides, this method offers the best search scalability for large
|
||||
data sets. Generally speaking, once the data set gets into the hundreds of thousands of vectors, this approach is
|
||||
preferred.
|
||||
|
||||
The k-NN plugin builds a native library index of the vectors for each "knn-vector field"/ "Lucene segment" pair during indexing that can be used to efficiently find the k-nearest neighbors to a query vector during search. To learn more about Lucene segments, see the [Apache Lucene documentation](https://lucene.apache.org/core/8_11_1/core/org/apache/lucene/codecs/lucene87/package-summary.html#package.description).
|
||||
These native library indices are loaded into native memory during search and managed by a cache. To learn more about
|
||||
pre-loading native library indices into memory, refer to the [warmup API]({{site.url}}{{site.baseurl}}/search-plugins/knn/api#warmup-operation). Additionally, you can see what native library indices are already loaded in memory, which you can learn more about in the [stats API section]({{site.url}}{{site.baseurl}}/search-plugins/knn/api#stats).
|
||||
|
||||
Because the graphs are constructed during indexing, it is not possible to apply a filter on an index and then use this search method. All filters are applied on the results produced by the approximate nearest neighbor search.
|
||||
Because the native library indices are constructed during indexing, it is not possible to apply a filter on an index
|
||||
and then use this search method. All filters are applied on the results produced by the approximate nearest neighbor
|
||||
search.
|
||||
|
||||
## Get started with approximate k-NN
|
||||
|
||||
To use the k-NN plugin's approximate search functionality, you must first create a k-NN index with setting `index.knn` to `true`. This setting tells the plugin to create HNSW graphs for the index.
|
||||
To use the k-NN plugin's approximate search functionality, you must first create a k-NN index with setting `index.knn`
|
||||
to `true`. This setting tells the plugin to create native library indices for the index.
|
||||
|
||||
Additionally, if you're using the approximate k-nearest neighbor method, specify `knn.space_type` to the space you're interested in. You can't change this setting after it's set. To see what spaces we support, see [spaces](#spaces). By default, `index.knn.space_type` is `l2`. For more information about index settings, such as algorithm parameters you can tweak to tune performance, see [Index settings]({{site.url}}{{site.baseurl}}/search-plugins/knn/knn-index#index-settings).
|
||||
|
||||
Next, you must add one or more fields of the `knn_vector` data type. This example creates an index with two `knn_vector` fields and uses cosine similarity:
|
||||
Next, you must add one or more fields of the `knn_vector` data type. This example creates an index with two
|
||||
`knn_vector`'s, one using *faiss*, the other using *nmslib*, fields:
|
||||
|
||||
```json
|
||||
PUT my-knn-index-1
|
||||
|
|
@ -38,7 +45,7 @@ PUT my-knn-index-1
|
|||
"properties": {
|
||||
"my_vector1": {
|
||||
"type": "knn_vector",
|
||||
"dimension": 4,
|
||||
"dimension": 2,
|
||||
"method": {
|
||||
"name": "hnsw",
|
||||
"space_type": "l2",
|
||||
|
|
@ -54,8 +61,8 @@ PUT my-knn-index-1
|
|||
"dimension": 4,
|
||||
"method": {
|
||||
"name": "hnsw",
|
||||
"space_type": "cosinesimil",
|
||||
"engine": "nmslib",
|
||||
"space_type": "innerproduct",
|
||||
"engine": "faiss",
|
||||
"parameters": {
|
||||
"ef_construction": 256,
|
||||
"m": 48
|
||||
|
|
@ -67,9 +74,14 @@ PUT my-knn-index-1
|
|||
}
|
||||
```
|
||||
|
||||
The `knn_vector` data type supports a vector of floats that can have a dimension of up to 10,000, as set by the dimension mapping parameter.
|
||||
In the example above, both `knn_vector`'s are configured from method definitions. Additionally, `knn_vector`'s can also
|
||||
be configured from models. Learn more about it [here]({{site.url}}{{site.baseurl}}/search-plugins/knn/knn-index#knn_vector-data-type)!
|
||||
|
||||
In OpenSearch, codecs handle the storage and retrieval of indices. The k-NN plugin uses a custom codec to write vector data to graphs so that the underlying k-NN search library can read it.
|
||||
The `knn_vector` data type supports a vector of floats that can have a dimension of up to 10,000, as set by the
|
||||
dimension mapping parameter.
|
||||
|
||||
In OpenSearch, codecs handle the storage and retrieval of indices. The k-NN plugin uses a custom codec to write vector
|
||||
data to native library indices so that the underlying k-NN search library can read it.
|
||||
{: .tip }
|
||||
|
||||
After you create the index, you can add some data to it:
|
||||
|
|
@ -114,10 +126,131 @@ GET my-knn-index-1/_search
|
|||
}
|
||||
```
|
||||
|
||||
`k` is the number of neighbors the search of each graph will return. You must also include the `size` option, which indicates how many results the query actually returns. The plugin returns `k` amount of results for each shard (and each segment) and `size` amount of results for the entire query. The plugin supports a maximum `k` value of 10,000.
|
||||
`k` is the number of neighbors the search of each graph will return. You must also include the `size` option, which
|
||||
indicates how many results the query actually returns. The plugin returns `k` amount of results for each shard
|
||||
(and each segment) and `size` amount of results for the entire query. The plugin supports a maximum `k` value of 10,000.
|
||||
|
||||
### Building a k-NN index from a model
|
||||
|
||||
For some of the algorithms that we support, the native library index needs to be trained before it can be used. Training
|
||||
everytime a segment is created would be very expensive, so, instead, we introduce the concept of a *model* that is used
|
||||
to initialize the native library index during segment creation. A *model* is created by calling the [Train API]({{site.url}}{{site.baseurl}}/search-plugins/knn/api#train-model),
|
||||
passing in the source of training data as well as the method definition of the model. Once training is complete, the
|
||||
model will be serialized to a k-NN model system index. Then, during indexing, the model is pulled from this index to
|
||||
initialize the segments.
|
||||
|
||||
In order to train a model, we first need an OpenSearch index with training data in it. Training data can come from
|
||||
any `knn_vector` field that has a dimension matching the dimension of the model you want to create. Training data can be
|
||||
the same data that you are going to index or a separate set. Let's create a training index:
|
||||
|
||||
```json
|
||||
PUT /train-index
|
||||
{
|
||||
"settings" : {
|
||||
"number_of_shards" : 3,
|
||||
"number_of_replicas" : 0
|
||||
},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"train-field": {
|
||||
"type": "knn_vector",
|
||||
"dimension": 4
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Notice that `index.knn` is not set in the index settings. This ensures that we do not create native library indices for
|
||||
this index.
|
||||
|
||||
Next, let's add some data to it:
|
||||
```json
|
||||
POST _bulk
|
||||
{ "index": { "_index": "train-index", "_id": "1" } }
|
||||
{ "train-field": [1.5, 5.5, 4.5, 6.4]}
|
||||
{ "index": { "_index": "train-index", "_id": "2" } }
|
||||
{ "train-field": [2.5, 3.5, 5.6, 6.7]}
|
||||
{ "index": { "_index": "train-index", "_id": "3" } }
|
||||
{ "train-field": [4.5, 5.5, 6.7, 3.7]}
|
||||
{ "index": { "_index": "train-index", "_id": "4" } }
|
||||
{ "train-field": [1.5, 5.5, 4.5, 6.4]}
|
||||
...
|
||||
```
|
||||
|
||||
After indexing into the training index completes, we can call our the Train API:
|
||||
```json
|
||||
POST /_plugins/_knn/models/_train/my-model
|
||||
{
|
||||
"training_index": "train-index",
|
||||
"training_field": "train-field",
|
||||
"dimension": 4,
|
||||
"description": "My models description",
|
||||
"search_size": 500,
|
||||
"method": {
|
||||
"name":"hnsw",
|
||||
"engine":"faiss",
|
||||
"parameters":{
|
||||
"encoder":{
|
||||
"name":"pq",
|
||||
"parameters":{
|
||||
"code_size": 8,
|
||||
"m": 8
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The Train API will return as soon as the training job is started. To check its status, we can use the Get Model API:
|
||||
```json
|
||||
GET /_plugins/_knn/models/my-model?filter_path=state&pretty
|
||||
{
|
||||
"state": "training"
|
||||
}
|
||||
```
|
||||
|
||||
Once the model enters the "created" state, we can create an index that will use this model to initialize it's native
|
||||
library indices:
|
||||
```json
|
||||
PUT /target-index
|
||||
{
|
||||
"settings" : {
|
||||
"number_of_shards" : 3,
|
||||
"number_of_replicas" : 1,
|
||||
"index.knn": true
|
||||
},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"target-field": {
|
||||
"type": "knn_vector",
|
||||
"model_id": "my-model"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Lastly, we can add the documents we want to be searched to the index:
|
||||
```json
|
||||
POST _bulk
|
||||
{ "index": { "_index": "target-index", "_id": "1" } }
|
||||
{ "target-field": [1.5, 5.5, 4.5, 6.4]}
|
||||
{ "index": { "_index": "target-index", "_id": "2" } }
|
||||
{ "target-field": [2.5, 3.5, 5.6, 6.7]}
|
||||
{ "index": { "_index": "target-index", "_id": "3" } }
|
||||
{ "target-field": [4.5, 5.5, 6.7, 3.7]}
|
||||
{ "index": { "_index": "target-index", "_id": "4" } }
|
||||
{ "target-field": [1.5, 5.5, 4.5, 6.4]}
|
||||
...
|
||||
```
|
||||
|
||||
After data is ingested, it can be search just like any other `knn_vector` field!
|
||||
|
||||
### Using approximate k-NN with filters
|
||||
If you use the `knn` query alongside filters or other clauses (e.g. `bool`, `must`, `match`), you might receive fewer than `k` results. In this example, `post_filter` reduces the number of results from 2 to 1:
|
||||
If you use the `knn` query alongside filters or other clauses (e.g. `bool`, `must`, `match`), you might receive fewer
|
||||
than `k` results. In this example, `post_filter` reduces the number of results from 2 to 1:
|
||||
|
||||
```json
|
||||
GET my-knn-index-1/_search
|
||||
|
|
@ -144,7 +277,12 @@ GET my-knn-index-1/_search
|
|||
|
||||
## Spaces
|
||||
|
||||
A space corresponds to the function used to measure the distance between two points in order to determine the k-nearest neighbors. From the k-NN perspective, a lower score equates to a closer and better result. This is the opposite of how OpenSearch scores results, where a greater score equates to a better result. To convert distances to OpenSearch scores, we take 1 / (1 + distance). Currently, the k-NN plugin supports the following spaces:
|
||||
A space corresponds to the function used to measure the distance between two points in order to determine the k-nearest
|
||||
neighbors. From the k-NN perspective, a lower score equates to a closer and better result. This is the opposite of how
|
||||
OpenSearch scores results, where a greater score equates to a better result. To convert distances to OpenSearch scores,
|
||||
we take 1 / (1 + distance). The k-NN plugin the spaces the plugin supports are below. Not every method supports each of
|
||||
these spaces. Be sure to check out [the method documentation]({{site.url}}{{site.baseurl}}/search-plugins/knn/knn-index#method-definitions) to make sure the space you are
|
||||
interested in is supported.
|
||||
|
||||
<table>
|
||||
<thead style="text-align: left">
|
||||
|
|
@ -183,5 +321,7 @@ A space corresponds to the function used to measure the distance between two poi
|
|||
</tr>
|
||||
</table>
|
||||
|
||||
The cosine similarity formula does not include the `1 -` prefix. However, because nmslib equates smaller scores with closer results, they return `1 - cosineSimilarity` for their cosine similarity space---that's why `1 -` is included in the distance function.
|
||||
The cosine similarity formula does not include the `1 -` prefix. However, because similarity search libraries equates
|
||||
smaller scores with closer results, they return `1 - cosineSimilarity` for cosine similarity space---that's why `1 -` is
|
||||
included in the distance function.
|
||||
{: .note }
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ This plugin supports three different methods for obtaining the k-nearest neighbo
|
|||
|
||||
1. **Approximate k-NN**
|
||||
|
||||
The first method takes an approximate nearest neighbor approach---it uses the HNSW algorithm to return the approximate k-nearest neighbors to a query vector. This algorithm sacrifices indexing speed and search accuracy in return for lower latency and more scalable search. To learn more about the algorithm, please refer to [nmslib's documentation](https://github.com/nmslib/nmslib/) or [the paper introducing the algorithm](https://arxiv.org/abs/1603.09320).
|
||||
The first method takes an approximate nearest neighbor approach---it uses one of several different algorithms to return the approximate k-nearest neighbors to a query vector. Usually, these algorithms sacrifice indexing speed and search accuracy in return for performance benefits such as lower latency, smaller memory footprints and more scalable search. To learn more about the algorithms, please refer to [*nmslib*](https://github.com/nmslib/nmslib/blob/master/manual/README.md)'s and [*faiss*](https://github.com/facebookresearch/faiss/wiki)'s documentation.
|
||||
|
||||
Approximate k-NN is the best choice for searches over large indices (i.e. hundreds of thousands of vectors or more) that require low latency. You should not use approximate k-NN if you want to apply a filter on the index before the k-NN search, which greatly reduces the number of vectors to be searched. In this case, you should use either the script scoring method or painless extensions.
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
---
|
||||
layout: default
|
||||
title: JNI libraries
|
||||
nav_order: 6
|
||||
parent: k-NN
|
||||
has_children: false
|
||||
---
|
||||
|
||||
# JNI libraries
|
||||
|
||||
To integrate [*nmslib*'s](https://github.com/nmslib/nmslib/) and [*faiss*'s](https://github.com/facebookresearch/faiss/) Approximate k-NN functionality (implemented in C++) into the k-NN plugin (implemented in Java), we created a Java Native Interface, which lets the k-NN plugin make calls to the native libraries. To implement this, we create 3 libraries: `libopensearchknn_nmslib`, the JNI library that interfaces with nmslib, `libopensearchknn_faiss`, the JNI library that interfaces with faiss, and `libopensearchknn_common`, a library containing common shared functionality between native libraries.
|
||||
|
||||
The libraries `libopensearchknn_faiss` and `libopensearchknn_nmslib` are lazily loaded when they are first called in the plugin. This means that if you are only planning on using one of the libraries, the other one will never be loaded.
|
||||
|
||||
For building the libraries from source, please refer to the [DEVELOPER_GUIDE](https://github.com/opensearch-project/k-NN/blob/main/DEVELOPER_GUIDE.md).
|
||||
|
||||
For more information about JNI, see [Java Native Interface](https://en.wikipedia.org/wiki/Java_Native_Interface) on Wikipedia.
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
---
|
||||
layout: default
|
||||
title: JNI library
|
||||
nav_order: 6
|
||||
parent: k-NN
|
||||
has_children: false
|
||||
---
|
||||
|
||||
# JNI library
|
||||
|
||||
To integrate [nmslib's](https://github.com/nmslib/nmslib/) approximate k-NN functionality (implemented in C++) into the k-NN plugin (implemented in Java), we created a Java Native Interface library, which lets the k-NN plugin leverage nmslib's functionality. To see how we build the JNI library binary and learn how to get the most of it in your production environment, see [JNI Library Artifacts](https://github.com/opensearch-project/k-NN#jni-library-artifacts).
|
||||
|
||||
For more information about JNI, see [Java Native Interface](https://en.wikipedia.org/wiki/Java_Native_Interface) on Wikipedia.
|
||||
|
|
@ -11,7 +11,14 @@ has_children: false
|
|||
## knn_vector data type
|
||||
|
||||
The k-NN plugin introduces a custom data type, the `knn_vector`, that allows users to ingest their k-NN vectors
|
||||
into an OpenSearch index.
|
||||
into an OpenSearch index and perform different kinds of k-NN search. The `knn_vector` field is highly configurable and
|
||||
can serve many different k-NN workloads. In general, a `knn_vector` field can be built either by providing a method
|
||||
definition or specifying a model id.
|
||||
|
||||
Method definitions are used when the underlying Approximate k-NN algorithm does not
|
||||
require training. For example, the following `knn_vector` field specifies that *nmslib*'s implementation of *hnsw*
|
||||
should be used for Approximate k-NN search. During indexing, *nmslib* will build the corresponding *hnsw* segment
|
||||
files.
|
||||
|
||||
```json
|
||||
"my_vector": {
|
||||
|
|
@ -27,19 +34,169 @@ into an OpenSearch index.
|
|||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Model id's are used when the underlying Approximate k-NN algorithm requires a training step. As a prerequisite, the
|
||||
model has to be created with the [Train API]({{site.url}}{{site.baseurl}}/search-plugins/knn/api#train-model). The
|
||||
model contains the information needed to initialize the native library segment files.
|
||||
|
||||
```json
|
||||
"type": "knn_vector",
|
||||
"model_id": "my-model"
|
||||
}
|
||||
```
|
||||
|
||||
Mapping Pararameter | Required | Default | Updateable | Description
|
||||
However, if you intend to just use painless scripting or a k-NN score script, you only need to pass the dimension.
|
||||
```json
|
||||
"type": "knn_vector",
|
||||
"dimension": 128
|
||||
}
|
||||
```
|
||||
|
||||
## Method Definitions
|
||||
|
||||
A method definition refers to the underlying configuration of the Approximate k-NN algorithm you want to use. Method
|
||||
definitions are used to either create a `knn_vector` field (when the method does not require training) or
|
||||
[create a model during training]({{site.url}}{{site.baseurl}}/search-plugins/knn/api#train-model) that can then be
|
||||
used to [create a `knn_vector` field]({{site.url}}{{site.baseurl}}/search-plugins/knn/approximate-knn/#building-a-k-nn-index-from-a-model).
|
||||
|
||||
A method definition will always contain the name of the method, the space_type the method is built for, the engine
|
||||
(the native library) to use, and a map of parameters.
|
||||
|
||||
Mapping Parameter | Required | Default | Updatable | Description
|
||||
:--- | :--- | :--- | :--- | :---
|
||||
`type` | true | n/a | false | The type of the field
|
||||
`dimension` | true | n/a | false | The vector dimension for the field
|
||||
`method` | false | null | false | The configuration for the Approximate nearest neighbor method
|
||||
`method.name` | true, if `method` is specified | n/a | false | The identifier for the nearest neighbor method. Currently, "hnsw" is the only valid method.
|
||||
`method.space_type` | false | "l2" | false | The vector space used to calculate the distance between vectors. Refer to [here]({{site.url}}{{site.baseurl}}/search-plugins/knn/approximate-knn#spaces)) to see available spaces.
|
||||
`method.engine` | false | "nmslib" | false | The approximate k-NN library to use for indexing and search. Currently, "nmslib" is the only valid engine.
|
||||
`method.parameters` | false | null | false | The parameters used for the nearest neighbor method.
|
||||
`method.parameters.ef_construction` | false | 512 | false | The size of the dynamic list used during k-NN graph creation. Higher values lead to a more accurate graph, but slower indexing speed. Only valid for "hnsw" method.
|
||||
`method.parameters.m` | false | 16 | false | The number of bidirectional links that the plugin creates for each new element. Increasing and decreasing this value can have a large impact on memory consumption. Keep this value between 2-100. Only valid for "hnsw" method
|
||||
`name` | true | n/a | false | The identifier for the nearest neighbor method.
|
||||
`space_type` | false | "l2" | false | The vector space used to calculate the distance between vectors.
|
||||
`engine` | false | "nmslib" | false | The approximate k-NN library to use for indexing and search. Either "faiss" or "nmslib".
|
||||
`parameters` | false | null | false | The parameters used for the nearest neighbor method.
|
||||
|
||||
### Supported nmslib methods
|
||||
|
||||
Method Name | Requires Training? | Supported Spaces | Description
|
||||
:--- | :--- | :--- | :---
|
||||
`hnsw` | false | "l2", "innerproduct", "cosinesimil", "l1", "linf" | Hierarchical proximity graph approach to Approximate k-NN search. For more details on the algorithm, [checkout this paper](https://arxiv.org/abs/1603.09320)!
|
||||
|
||||
#### HNSW Parameters
|
||||
|
||||
Paramater Name | Required | Default | Updatable | Description
|
||||
:--- | :--- | :--- | :--- | :---
|
||||
`ef_construction` | false | 512 | false | The size of the dynamic list used during k-NN graph creation. Higher values lead to a more accurate graph, but slower indexing speed.
|
||||
`m` | false | 16 | false | The number of bidirectional links that the plugin creates for each new element. Increasing and decreasing this value can have a large impact on memory consumption. Keep this value between 2-100.
|
||||
|
||||
**Note** --- For *nmslib*, *ef_search* is set in the [index settings](#index-settings).
|
||||
|
||||
### Supported faiss methods
|
||||
|
||||
Method Name | Requires Training? | Supported Spaces | Description
|
||||
:--- | :--- | :--- | :---
|
||||
`hnsw` | false | "l2", "innerproduct"* | Hierarchical proximity graph approach to Approximate k-NN search.
|
||||
`ivf` | true | "l2", "innerproduct" | Bucketing approach where vectors are assigned different buckets based on clustering and, during search, only a subset of the buckets are searched.
|
||||
|
||||
**Note** --- For *hnsw*, "innerproduct" is not available when PQ is used.
|
||||
|
||||
#### HNSW Parameters
|
||||
|
||||
Paramater Name | Required | Default | Updatable | Description
|
||||
:--- | :--- | :--- | :--- | :---
|
||||
`ef_search` | false | 512 | false | The size of the dynamic list used during k-NN searches. Higher values lead to more accurate but slower searches.
|
||||
`ef_construction` | false | 512 | false | The size of the dynamic list used during k-NN graph creation. Higher values lead to a more accurate graph, but slower indexing speed.
|
||||
`m` | false | 16 | false | The number of bidirectional links that the plugin creates for each new element. Increasing and decreasing this value can have a large impact on memory consumption. Keep this value between 2-100.
|
||||
`encoder` | false | flat | false | Encoder definition for encoding vectors. Encoders can reduce the memory footprint of your index, at the expense of search accuracy.
|
||||
|
||||
#### IVF Parameters
|
||||
|
||||
Paramater Name | Required | Default | Updatable | Description
|
||||
:--- | :--- | :--- | :--- | :---
|
||||
`nlists` | false | 4 | false | Number of buckets to partition vectors into. Higher values may lead to more accurate searches, at the expense of memory and training latency. For more information about choosing the right value, refer to [*faiss*'s documentation](https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index).
|
||||
`nprobes` | false | 1 | false | Number of buckets to search over during query. Higher values lead to more accurate but slower searches.
|
||||
`encoder` | false | flat | false | Encoder definition for encoding vectors. Encoders can reduce the memory footprint of your index, at the expense of search accuracy.
|
||||
|
||||
For more information about setting these parameters, please refer to [*faiss*'s documentation](https://github.com/facebookresearch/faiss/wiki/Faiss-indexes).
|
||||
|
||||
#### IVF training requirements
|
||||
|
||||
The IVF algorithm requires a training step. To create an index that uses IVF, you need to train a model with the
|
||||
[Train API]({{site.url}}{{site.baseurl}}/search-plugins/knn/api#train-model), passing the IVF method definition. IVF requires that, at a minimum, there should be `nlist` training
|
||||
data points, but it is [recommended to use more](https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index#how-big-is-the-dataset).
|
||||
Training data can either the same data that is going to be ingested or a separate set of data.
|
||||
|
||||
### Supported faiss encoders
|
||||
|
||||
Encoders can be used to reduce the memory footprint of a k-NN index at the expense of search accuracy. *faiss* has
|
||||
several different encoder types, but currently, the plugin only supports *flat* and *pq* encoding.
|
||||
|
||||
An example method definition that specifies an encoder may look something like this:
|
||||
```json
|
||||
"method": {
|
||||
"name":"hnsw",
|
||||
"engine":"faiss",
|
||||
"parameters":{
|
||||
"encoder":{
|
||||
"name":"pq",
|
||||
"parameters":{
|
||||
"code_size": 8,
|
||||
"m": 8
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Encoder Name | Requires Training? | Description
|
||||
:--- | :--- | :---
|
||||
`flat` | false | Encode vectors as floating point arrays. This encoding does not reduce memory footprint.
|
||||
`pq` | true | Short for product quantization, it is a lossy compression technique that encodes a vector into a fixed size of bytes using clustering, with the goal of minimizing the drop in k-NN search accuracy. From a high level, vectors are broken up into `m` subvectors, and then each subvector is represented by a `code_size` code obtained from a code book produced during training. For more details on product quantization, here is a [great blog post](https://medium.com/dotstar/understanding-faiss-part-2-79d90b1e5388)!
|
||||
|
||||
#### PQ Parameters
|
||||
|
||||
Paramater Name | Required | Default | Updatable | Description
|
||||
:--- | :--- | :--- | :--- | :---
|
||||
`m` | false | 1 | false | Determine how many many sub-vectors to break the vector into. sub-vectors are encoded independently of each other. This dimension of the vector must be divisible by `m`. Max value is 1024.
|
||||
`code_size` | false | 8 | false | Determines the number of bits to encode a sub-vector into. Max value is 8. **Note** --- for IVF, this value must be less than or equal to 8. For HNSW, this value can only be 8.
|
||||
|
||||
### Choosing the right method
|
||||
|
||||
There are a lot of options to choose from when building your `knn_vector` field. To determine the correct methods and
|
||||
parameters to choose, you should first understand what requirements you have for your workload and what trade-offs you
|
||||
are willing to make. Factors to consider are (1) query latency, (2) query quality, (3) memory limits,
|
||||
(4) indexing latency.
|
||||
|
||||
If memory is not a concern, HNSW offers a very strong query latency/query quality tradeoff.
|
||||
|
||||
If you want to use less memory and index faster than HNSW, while maintaining similar query quality, you should evaluate IVF.
|
||||
|
||||
If memory is a concern, consider adding a PQ encoder to your HNSW or IVF index. Because PQ is a lossy encoding, query
|
||||
quality will drop.
|
||||
|
||||
### Memory Estimation
|
||||
|
||||
In a typical OpenSearch cluster, a certain portion of RAM is set aside for the JVM heap. The k-NN plugin allocates
|
||||
native library indices to a portion of the remaining RAM. This portion's size is determined by
|
||||
the `circuit_breaker_limit` cluster setting. By default, the limit is set at 50%.
|
||||
|
||||
Having a replica doubles the total number of vectors.
|
||||
{: .note }
|
||||
|
||||
#### HNSW memory estimation
|
||||
|
||||
The memory required for HNSW is estimated to be `1.1 * (4 * dimension + 8 * M)` bytes/vector.
|
||||
|
||||
As an example, assume you have a million vectors with a dimension of 256 and M of 16. The memory requirement can be estimated as follows:
|
||||
|
||||
```
|
||||
1.1 * (4 *256 + 8 * 16) * 1,000,000 ~= 1.26 GB
|
||||
```
|
||||
|
||||
#### IVF memory estimation
|
||||
|
||||
The memory required for IVF is estimated to be `1.1 * (((4 * dimension) * num_vectors) + (4 * nlist * d))` bytes.
|
||||
|
||||
As an example, assume you have a million vectors with a dimension of 256 and nlist of 128. The memory requirement can be estimated as follows:
|
||||
|
||||
```
|
||||
1.1 * (((4 * 128) * 1000000) + (4 * 128 * 256)) ~= 563 MB
|
||||
|
||||
```
|
||||
|
||||
## Index settings
|
||||
|
||||
|
|
@ -52,8 +209,8 @@ different parameters.
|
|||
|
||||
Setting | Default | Updateable | Description
|
||||
:--- | :--- | :--- | :---
|
||||
`index.knn` | false | false | Whether the index should build hnsw graphs for the `knn_vector` fields. If set to false, the `knn_vector` fields will be stored in doc values, but Approximate k-NN search functionality will be disabled.
|
||||
`index.knn.algo_param.ef_search` | 512 | true | The size of the dynamic list used during k-NN searches. Higher values lead to more accurate but slower searches.
|
||||
`index.knn.algo_param.ef_construction` | 512 | false | (Deprecated in 1.0.0. Use the mapping parameters to set this value instead.) Refer to mapping definition.
|
||||
`index.knn.algo_param.m` | 16 | false | (Deprecated in 1.0.0. Use the mapping parameters to set this value instead.) Refer to mapping definition.
|
||||
`index.knn.space_type` | "l2" | false | (Deprecated in 1.0.0. Use the mapping parameters to set this value instead.) Refer to mapping definition.
|
||||
`index.knn` | false | false | Whether the index should build native library indices for the `knn_vector` fields. If set to false, the `knn_vector` fields will be stored in doc values, but Approximate k-NN search functionality will be disabled.
|
||||
`index.knn.algo_param.ef_search` | 512 | true | The size of the dynamic list used during k-NN searches. Higher values lead to more accurate but slower searches. Only available for *nmslib*.
|
||||
`index.knn.algo_param.ef_construction` | 512 | false | (Deprecated in 1.0.0. Use the mapping parameters to set this value instead.) Only available for *nmslib*. Refer to mapping definition.
|
||||
`index.knn.algo_param.m` | 16 | false | (Deprecated in 1.0.0. Use the mapping parameters to set this value instead.) Only available for *nmslib*. Refer to mapping definition.
|
||||
`index.knn.space_type` | "l2" | false | (Deprecated in 1.0.0. Use the mapping parameters to set this value instead.) Only available for *nmslib*. Refer to mapping definition.
|
||||
|
|
|
|||
|
|
@ -8,9 +8,9 @@ nav_order: 8
|
|||
# Performance tuning
|
||||
|
||||
This topic provides performance tuning recommendations to improve indexing and search performance for approximate k-NN. From a high level, k-NN works according to these principles:
|
||||
* Graphs are created per knn_vector field / (Lucene) segment pair.
|
||||
* Native library indices are created per knn_vector field / (Lucene) segment pair.
|
||||
* Queries execute on segments sequentially inside the shard (same as any other OpenSearch query).
|
||||
* Each graph in the segment returns <=k neighbors.
|
||||
* Each native library index in the segment returns <=k neighbors.
|
||||
* The coordinator node picks up final size number of neighbors from the neighbors returned by each shard.
|
||||
|
||||
This topic also provides recommendations for comparing approximate k-NN to exact k-NN with score script.
|
||||
|
|
@ -35,13 +35,13 @@ Take the following steps to improve indexing performance, especially when you pl
|
|||
|
||||
* **Disable replicas (no OpenSearch replica shard)**
|
||||
|
||||
Set replicas to `0` to prevent duplicate construction of graphs in both primary and replica shards. When you enable replicas after indexing finishes, the serialized graphs are directly copied. If you have no replicas, losing nodes might cause data loss, so it's important that the data lives elsewhere so this initial load can be retried in case of an issue.
|
||||
Set replicas to `0` to prevent duplicate construction of native library indices in both primary and replica shards. When you enable replicas after indexing finishes, the serialized native library indices are directly copied. If you have no replicas, losing nodes might cause data loss, so it's important that the data lives elsewhere so this initial load can be retried in case of an issue.
|
||||
|
||||
* **Increase the number of indexing threads**
|
||||
|
||||
If the hardware you choose has multiple cores, you can allow multiple threads in graph construction by speeding up the indexing process. Determine the number of threads to allot with the [knn.algo_param.index_thread_qty]({{site.url}}{{site.baseurl}}/search-plugins/knn/settings#cluster-settings) setting.
|
||||
If the hardware you choose has multiple cores, you can allow multiple threads in native library index construction by speeding up the indexing process. Determine the number of threads to allot with the [knn.algo_param.index_thread_qty]({{site.url}}{{site.baseurl}}/search-plugins/knn/settings#cluster-settings) setting.
|
||||
|
||||
Keep an eye on CPU utilization and choose the correct number of threads. Because graph construction is costly, having multiple threads can cause additional CPU load.
|
||||
Keep an eye on CPU utilization and choose the correct number of threads. Because native library index construction is costly, having multiple threads can cause additional CPU load.
|
||||
|
||||
## Search performance tuning
|
||||
|
||||
|
|
@ -49,7 +49,7 @@ Take the following steps to improve search performance:
|
|||
|
||||
* **Reduce segment count**
|
||||
|
||||
To improve search performance, you must keep the number of segments under control. Lucene's IndexSearcher searches over all of the segments in a shard to find the 'size' best results. However, because the complexity of search for the HNSW algorithm is logarithmic with respect to the number of vectors, searching over five graphs with 100 vectors each and then taking the top 'size' results from 5*k results will take longer than searching over one graph with 500 vectors and then taking the top size results from k results.
|
||||
To improve search performance, you must keep the number of segments under control. Lucene's IndexSearcher searches over all of the segments in a shard to find the 'size' best results.
|
||||
|
||||
Ideally, having one segment per shard provides the optimal performance with respect to search latency. You can configure an index to have multiple shards to avoid giant shards and achieve more parallelism.
|
||||
|
||||
|
|
@ -57,9 +57,9 @@ Take the following steps to improve search performance:
|
|||
|
||||
* **Warm up the index**
|
||||
|
||||
Graphs are constructed during indexing, but they're loaded into memory during the first search. In Lucene, each segment is searched sequentially (so, for k-NN, each segment returns up to k nearest neighbors of the query point), and the top 'size' number of results based on the score are returned from all the results returned by segements at a shard level (higher score = better result).
|
||||
Native library indices are constructed during indexing, but they're loaded into memory during the first search. In Lucene, each segment is searched sequentially (so, for k-NN, each segment returns up to k nearest neighbors of the query point), and the top 'size' number of results based on the score are returned from all the results returned by segements at a shard level (higher score = better result).
|
||||
|
||||
Once a graph is loaded (graphs are loaded outside OpenSearch JVM), OpenSearch caches them in memory. Initial queries are expensive and take a few seconds, while subsequent queries are faster and take milliseconds (assuming the k-NN circuit breaker isn't hit).
|
||||
Once a native library index is loaded (native library indices are loaded outside OpenSearch JVM), OpenSearch caches them in memory. Initial queries are expensive and take a few seconds, while subsequent queries are faster and take milliseconds (assuming the k-NN circuit breaker isn't hit).
|
||||
|
||||
To avoid this latency penalty during your first queries, you can use the warmup API operation on the indices you want to search:
|
||||
|
||||
|
|
@ -74,9 +74,9 @@ Take the following steps to improve search performance:
|
|||
}
|
||||
```
|
||||
|
||||
The warmup API operation loads all graphs for all shards (primary and replica) for the specified indices into the cache, so there's no penalty to load graphs during initial searches.
|
||||
The warmup API operation loads all native library indices for all shards (primary and replica) for the specified indices into the cache, so there's no penalty to load native library indices during initial searches.
|
||||
|
||||
**Note**: This API operation only loads the segments of the indices it ***sees*** into the cache. If a merge or refresh operation finishes after the API runs, or if you add new documents, you need to rerun the API to load those graphs into memory.
|
||||
**Note**: This API operation only loads the segments of the indices it ***sees*** into the cache. If a merge or refresh operation finishes after the API runs, or if you add new documents, you need to rerun the API to load those native library indices into memory.
|
||||
|
||||
* **Avoid reading stored fields**
|
||||
|
||||
|
|
@ -84,26 +84,9 @@ Take the following steps to improve search performance:
|
|||
|
||||
## Improving recall
|
||||
|
||||
Recall depends on multiple factors like number of vectors, number of dimensions, segments, and so on. Searching over a large number of small segments and aggregating the results leads to better recall than searching over a small number of large segments and aggregating results. The larger the graph, the more chances of losing recall if you're using smaller algorithm parameters. Choosing larger values for algorithm parameters should help solve this issue but sacrifices search latency and indexing time. That being said, it's important to understand your system's requirements for latency and accuracy, and then choose the number of segments you want your index to have based on experimentation.
|
||||
Recall depends on multiple factors like number of vectors, number of dimensions, segments, and so on. Searching over a large number of small segments and aggregating the results leads to better recall than searching over a small number of large segments and aggregating results. The larger the native library index, the more chances of losing recall if you're using smaller algorithm parameters. Choosing larger values for algorithm parameters should help solve this issue but sacrifices search latency and indexing time. That being said, it's important to understand your system's requirements for latency and accuracy, and then choose the number of segments you want your index to have based on experimentation.
|
||||
|
||||
To configure recall, adjust the algorithm parameters of the HNSW algorithm exposed through index settings. Algorithm parameters that control recall are `m`, `ef_construction`, and `ef_search`. For more information about how algorithm parameters influence indexing and search recall, see [HNSW algorithm parameters](https://github.com/nmslib/hnswlib/blob/master/ALGO_PARAMS.md). Increasing these values can help recall and lead to better search results, but at the cost of higher memory utilization and increased indexing time.
|
||||
|
||||
The default recall values work on a broader set of use cases, but make sure to run your own experiments on your data sets and choose the appropriate values. For index-level settings, see [Index settings]({{site.url}}{{site.baseurl}}/search-plugins/knn/knn-index#index-settings).
|
||||
|
||||
## Estimating memory usage
|
||||
|
||||
In a typical OpenSearch cluster, a certain portion of RAM is set aside for the JVM heap. The k-NN plugin allocates graphs to a portion of the remaining RAM. This portion's size is determined by the `circuit_breaker_limit` cluster setting. By default, the limit is set at 50%.
|
||||
|
||||
The memory required for graphs is estimated to be `1.1 * (4 * dimension + 8 * M)` bytes/vector.
|
||||
|
||||
As an example, assume you have a million vectors with a dimension of 256 and M of 16. The memory requirement can be estimated as follows:
|
||||
|
||||
```
|
||||
1.1 * (4 *256 + 8 * 16) * 1,000,000 ~= 1.26 GB
|
||||
```
|
||||
|
||||
Having a replica doubles the total number of vectors.
|
||||
{: .note }
|
||||
The default parameters work on a broader set of use cases, but make sure to run your own experiments on your data sets and choose the appropriate values. For index-level settings, see [Index settings]({{site.url}}{{site.baseurl}}/search-plugins/knn/knn-index#index-settings).
|
||||
|
||||
## Approximate nearest neighbor versus score script
|
||||
|
||||
|
|
|
|||
|
|
@ -13,11 +13,13 @@ The k-NN plugin adds several new cluster settings.
|
|||
|
||||
Setting | Default | Description
|
||||
:--- | :--- | :---
|
||||
`knn.algo_param.index_thread_qty` | 1 | The number of threads used for graph creation. Keeping this value low reduces the CPU impact of the k-NN plugin, but also reduces indexing performance.
|
||||
`knn.cache.item.expiry.enabled` | false | Whether to remove graphs that have not been accessed for a certain duration from memory.
|
||||
`knn.cache.item.expiry.minutes` | 3h | If enabled, the idle time before removing a graph from memory.
|
||||
`knn.circuit_breaker.unset.percentage` | 75.0 | The native memory usage threshold for the circuit breaker. Memory usage must be below this percentage of `knn.memory.circuit_breaker.limit` for `knn.circuit_breaker.triggered` to remain false.
|
||||
`knn.algo_param.index_thread_qty` | 1 | The number of threads used for native library index creation. Keeping this value low reduces the CPU impact of the k-NN plugin, but also reduces indexing performance.
|
||||
`knn.cache.item.expiry.enabled` | false | Whether to remove native library indices that have not been accessed for a certain duration from memory.
|
||||
`knn.cache.item.expiry.minutes` | 3h | If enabled, the idle time before removing a native library index from memory.
|
||||
`knn.circuit_breaker.unset.percentage` | 75% | The native memory usage threshold for the circuit breaker. Memory usage must be below this percentage of `knn.memory.circuit_breaker.limit` for `knn.circuit_breaker.triggered` to remain false.
|
||||
`knn.circuit_breaker.triggered` | false | True when memory usage exceeds the `knn.circuit_breaker.unset.percentage` value.
|
||||
`knn.memory.circuit_breaker.limit` | 50% | The native memory limit for graphs. At the default value, if a machine has 100 GB of memory and the JVM uses 32 GB, the k-NN plugin uses 50% of the remaining 68 GB (34 GB). If memory usage exceeds this value, k-NN removes the least recently used graphs.
|
||||
`knn.memory.circuit_breaker.limit` | 50% | The native memory limit for native library indices. At the default value, if a machine has 100 GB of memory and the JVM uses 32 GB, the k-NN plugin uses 50% of the remaining 68 GB (34 GB). If memory usage exceeds this value, k-NN removes the least recently used native library indices.
|
||||
`knn.memory.circuit_breaker.enabled` | true | Whether to enable the k-NN memory circuit breaker.
|
||||
`knn.plugin.enabled`| true | Enables or disables the k-NN plugin.
|
||||
`knn.model.index.number_of_shards`| 1 | Number of shards to use for the model system index, the OpenSearch index that stores the models used for Approximate k-NN Search.
|
||||
`knn.model.index.number_of_replicas`| 1 | Number of replica shards to use for the model system index. Generally, in a multi-node cluster, this should be at least 1 to increase stability.
|
||||
|
|
|
|||
|
|
@ -13,11 +13,97 @@ To learn about full-text queries in OpenSearch, see [Full-text queries]({{site.u
|
|||
|
||||
## Match
|
||||
|
||||
Use the `match` command to search documents that match a `string`, `number`, `date`, or `boolean` value for a given field.
|
||||
|
||||
### Syntax
|
||||
|
||||
```sql
|
||||
match(field_expression, query_expression[, option=<option_value>]*)
|
||||
```
|
||||
|
||||
You can specify the following options:
|
||||
|
||||
- `analyzer`
|
||||
- `auto_generate_synonyms_phrase`
|
||||
- `fuzziness`
|
||||
- `max_expansions`
|
||||
- `prefix_length`
|
||||
- `fuzzy_transpositions`
|
||||
- `fuzzy_rewrite`
|
||||
- `lenient`
|
||||
- `operator`
|
||||
- `minimum_should_match`
|
||||
- `zero_terms_query`
|
||||
- `boost`
|
||||
|
||||
*Example 1*: Search the `message` field:
|
||||
|
||||
```json
|
||||
GET my_index/_search
|
||||
{
|
||||
"query": {
|
||||
"match": {
|
||||
"message": "this is a test"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
SQL query:
|
||||
|
||||
```sql
|
||||
SELECT message FROM my_index WHERE match(message, "this is a test")
|
||||
```
|
||||
|
||||
*Example 2*: Search the `message` field with the `operator` parameter:
|
||||
|
||||
```json
|
||||
GET my_index/_search
|
||||
{
|
||||
"query": {
|
||||
"match": {
|
||||
"message": {
|
||||
"query": "this is a test",
|
||||
"operator": "and"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
SQL query:
|
||||
|
||||
```sql
|
||||
SELECT message FROM my_index WHERE match(message, "this is a test", operator=and)
|
||||
```
|
||||
|
||||
*Example 3*: Search the `message` field with the `operator` and `zero_terms_query` parameters:
|
||||
|
||||
```json
|
||||
GET my_index/_search
|
||||
{
|
||||
"query": {
|
||||
"match": {
|
||||
"message": {
|
||||
"query": "to be or not to be",
|
||||
"operator": "and",
|
||||
"zero_terms_query": "all"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
SQL query:
|
||||
|
||||
```sql
|
||||
SELECT message FROM my_index WHERE match(message, "this is a test", operator=and, zero_terms_query=all)
|
||||
```
|
||||
|
||||
To search for text in a single field, use `MATCHQUERY` or `MATCH_QUERY` functions.
|
||||
|
||||
Pass in your search query and the field name that you want to search against.
|
||||
|
||||
|
||||
```sql
|
||||
SELECT account_number, address
|
||||
FROM accounts
|
||||
|
|
|
|||
|
|
@ -151,8 +151,8 @@ _meta:
|
|||
The open source version of OpenSearch Dashboards saves all objects to a single index: `.opensearch-dashboards`. The security plugin uses this index for the global tenant, but separate indices for every other tenant. Each user also has a private tenant, so you might see a large number of indices that follow two patterns:
|
||||
|
||||
```
|
||||
.opensearch_dashboards_<hash>_<tenant_name>
|
||||
.opensearch_dashboards_<hash>_<username>
|
||||
.kibana_<hash>_<tenant_name>
|
||||
.kibana_<hash>_<username>
|
||||
```
|
||||
|
||||
The security plugin scrubs these index names of special characters, so they might not be a perfect match of tenant names and usernames.
|
||||
|
|
|
|||
|
|
@ -80,8 +80,10 @@ These permissions are for the cluster and can't be applied granularly. For examp
|
|||
- cluster:admin/opendistro/ad/detector/search
|
||||
- cluster:admin/opendistro/ad/detector/stats
|
||||
- cluster:admin/opendistro/ad/detector/write
|
||||
- cluster:admin/opendistro/ad/detector/validate
|
||||
- cluster:admin/opendistro/ad/detectors/get
|
||||
- cluster:admin/opendistro/ad/result/search
|
||||
- cluster:admin/opendistro/ad/result/topAnomalies
|
||||
- cluster:admin/opendistro/ad/tasks/search
|
||||
- cluster:admin/opendistro/alerting/alerts/ack (acknowledge)
|
||||
- cluster:admin/opendistro/alerting/alerts/get
|
||||
|
|
@ -130,6 +132,7 @@ These permissions are for the cluster and can't be applied granularly. For examp
|
|||
- cluster:admin/opendistro/reports/instance/list
|
||||
- cluster:admin/opendistro/reports/instance/get
|
||||
- cluster:admin/opendistro/reports/menu/download
|
||||
- cluster:admin/plugins/replication/autofollow/update
|
||||
- cluster:admin/reindex/rethrottle
|
||||
- cluster:admin/repository/delete
|
||||
- cluster:admin/repository/get
|
||||
|
|
@ -196,6 +199,13 @@ These permissions apply to an index or index pattern. You might want a user to h
|
|||
- indices:admin/mappings/fields/get*
|
||||
- indices:admin/mappings/get
|
||||
- indices:admin/open
|
||||
- indices:admin/plugins/replication/index/setup/validate
|
||||
- indices:admin/plugins/replication/index/start
|
||||
- indices:admin/plugins/replication/index/pause
|
||||
- indices:admin/plugins/replication/index/resume
|
||||
- indices:admin/plugins/replication/index/stop
|
||||
- indices:admin/plugins/replication/index/update
|
||||
- indices:admin/plugins/replication/index/status_check
|
||||
- indices:admin/refresh
|
||||
- indices:admin/refresh*
|
||||
- indices:admin/resolve/index
|
||||
|
|
@ -221,6 +231,8 @@ These permissions apply to an index or index pattern. You might want a user to h
|
|||
- indices:data/read/msearch/template
|
||||
- indices:data/read/mtv (multi-term vectors)
|
||||
- indices:data/read/mtv*
|
||||
- indices:data/read/plugins/replication/file_chunk
|
||||
- indices:data/read/plugins/replication/changes
|
||||
- indices:data/read/scroll
|
||||
- indices:data/read/scroll/clear
|
||||
- indices:data/read/search
|
||||
|
|
@ -231,6 +243,7 @@ These permissions apply to an index or index pattern. You might want a user to h
|
|||
- indices:data/write/bulk*
|
||||
- indices:data/write/delete (delete documents)
|
||||
- indices:data/write/delete/byquery
|
||||
- indices:data/write/plugins/replication/changes
|
||||
- indices:data/write/index (add documents to existing indices)
|
||||
- indices:data/write/reindex
|
||||
- indices:data/write/update
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue