From d31b21082db53c6ba228ec8c8899f2f457ec846e Mon Sep 17 00:00:00 2001 From: Xue Zhou Date: Mon, 19 Jul 2021 23:16:03 -0700 Subject: [PATCH 001/229] Update configuration.md --- _opensearch/configuration.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/_opensearch/configuration.md b/_opensearch/configuration.md index a6a0f995..726591a2 100755 --- a/_opensearch/configuration.md +++ b/_opensearch/configuration.md @@ -65,4 +65,8 @@ PUT _cluster/settings You can find `opensearch.yml` in `/usr/share/opensearch/config/opensearch.yml` (Docker) or `/etc/opensearch/opensearch.yml` (most Linux distributions) on each node. +You can edit the `OPENSEARCH_PATH_CONF=/etc/opensearch` to change the config directory location. This variable is sourced from `/etc/default/opensearch`(Debian package) and `/etc/sysconfig/opensearch`(RPM package). + +If you set your customized `OPENSEARCH_PATH_CONF` variable, be aware that other default environment variables will not be loaded. + The demo configuration includes a number of settings for the security plugin that you should modify before using OpenSearch for a production workload. To learn more, see [Security]({{site.url}}{{site.baseurl}}/security-plugin/). From 305266b1dc158247ef81e70d8bff283c7302f449 Mon Sep 17 00:00:00 2001 From: Xue Zhou Date: Mon, 19 Jul 2021 23:57:13 -0700 Subject: [PATCH 002/229] =?UTF-8?q?=E2=80=9Cconfiguration-change=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Xue Zhou --- _im-plugin/ism/api.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/_im-plugin/ism/api.md b/_im-plugin/ism/api.md index 20646487..4b568ad6 100644 --- a/_im-plugin/ism/api.md +++ b/_im-plugin/ism/api.md @@ -160,6 +160,10 @@ POST _plugins/_ism/add/index_1 } ``` +If you use a wildcard `*` while adding a policy to an index, the ISM plugin interprets `*` as all indices, including system indices like `.opendistro-security`, which stores users, roles, and tenants. A delete action in your policy might accidentally delete all user roles and tenants in your cluster. +Don't use the broad `*` wildcard, and instead add a prefix, such as `my-logs*`, when specifying indices with the `_ism/add` API. +{: .warning } + --- From 0bf8624824ed4954e2ad3eda2464552c5aab3c5e Mon Sep 17 00:00:00 2001 From: aetter Date: Thu, 19 Aug 2021 12:56:53 -0700 Subject: [PATCH 003/229] Getting started content for Python --- _clients/java-rest-high-level.md | 2 +- _clients/python.md | 122 +++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 _clients/python.md diff --git a/_clients/java-rest-high-level.md b/_clients/java-rest-high-level.md index 1dcf1563..3f7260cf 100644 --- a/_clients/java-rest-high-level.md +++ b/_clients/java-rest-high-level.md @@ -1,7 +1,7 @@ --- layout: default title: Java high-level REST client -nav_order: 97 +nav_order: 60 --- # Java high-level REST client diff --git a/_clients/python.md b/_clients/python.md new file mode 100644 index 00000000..1a5bf9b0 --- /dev/null +++ b/_clients/python.md @@ -0,0 +1,122 @@ +--- +layout: default +title: Python client +nav_order: 70 +--- + +# Python client + +The OpenSearch Python client provides a more natural syntax for interacting with your cluster. Rather than sending HTTP requests raw JSON bodies to a given URL, you can create an OpenSearch client for your cluster and call the client's built-in functions. + + +## Setup + +To add the client to your project, install it using pip: + +```bash +pip install +``` + +Then import it like any other module: + +```python +from opensearch import OpenSearch +``` + + +## Sample code + +```python +from opensearch import OpenSearch + +host = 'localhost' +port = 9200 +auth = ('admin', 'admin') # For testing only. Don't store credentials in code. +ca_certs_path = '/full/path/to/root-ca.pem' # Provide a CA bundle if you use intermediate CAs with your root CA + +# Optional client certificates if you don't want to use HTTP basic authentication. +# client_cert_path = '/full/path/to/client.pem' +# client_key_path = '/full/path/to/client-key.pem' + +# Create the client with SSL/TLS enabled, but hostname verification disabled. +client = OpenSearch( + hosts = [{'host': host, 'port': port}], + http_compress=True, # enables gzip compression for request bodies + http_auth = auth, + # client_cert = client_cert_path, + # client_key = client_key_path, + use_ssl = True, + verify_certs = True, + ssl_assert_hostname = False, + ssl_show_warn = False, + ca_certs = ca_certs_path +) + +# Create an index with non-default settings. +index_name = 'python-test-index3' +index_body = { + 'settings': { + 'index': { + 'number_of_shards': 4 + } + } +} + +response = client.indices.create(index_name, body=index_body) +print('\nCreating index:') +print(response) + +# Add a document to the index. +document = { + 'title': 'Moneyball', + 'director': 'Bennett Miller', + 'year': '2011' +} +id = '1' + +response = client.index( + index=index_name, + body=document, + id=id, + refresh=True +) + +print('\nAdding document:') +print(response) + +# Search for the document. +q = 'miller' +query = { + 'size': 5, + 'query': { + 'multi_match': { + 'query': q, + 'fields': ['title^2', 'director'] + } + } +} + +response = client.search( + body=query, + index=index_name +) +print('\nSearch results:') +print(response) + +# Delete the document. +response = client.delete( + index=index_name, + id=id +) + +print('\nDeleting document:') +print(response) + +# Delete the index. +response = client.indices.delete( + index=index_name +) + +print('\nDeleting index:') +print(response) +``` From 9ce5d95786a3baf50be225cdf6c44a3943ff2988 Mon Sep 17 00:00:00 2001 From: aetter Date: Thu, 19 Aug 2021 13:23:14 -0700 Subject: [PATCH 004/229] Clean up spacing --- _clients/python.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/_clients/python.md b/_clients/python.md index 1a5bf9b0..0a44cfa9 100644 --- a/_clients/python.md +++ b/_clients/python.md @@ -32,7 +32,7 @@ from opensearch import OpenSearch host = 'localhost' port = 9200 auth = ('admin', 'admin') # For testing only. Don't store credentials in code. -ca_certs_path = '/full/path/to/root-ca.pem' # Provide a CA bundle if you use intermediate CAs with your root CA +ca_certs_path = '/full/path/to/root-ca.pem' # Provide a CA bundle if you use intermediate CAs with your root CA. # Optional client certificates if you don't want to use HTTP basic authentication. # client_cert_path = '/full/path/to/client.pem' @@ -41,7 +41,7 @@ ca_certs_path = '/full/path/to/root-ca.pem' # Provide a CA bundle if you use int # Create the client with SSL/TLS enabled, but hostname verification disabled. client = OpenSearch( hosts = [{'host': host, 'port': port}], - http_compress=True, # enables gzip compression for request bodies + http_compress = True, # enables gzip compression for request bodies http_auth = auth, # client_cert = client_cert_path, # client_key = client_key_path, @@ -75,10 +75,10 @@ document = { id = '1' response = client.index( - index=index_name, - body=document, - id=id, - refresh=True + index = index_name, + body = document, + id = id, + refresh = True ) print('\nAdding document:') @@ -97,16 +97,16 @@ query = { } response = client.search( - body=query, - index=index_name + body = query, + index = index_name ) print('\nSearch results:') print(response) # Delete the document. response = client.delete( - index=index_name, - id=id + index = index_name, + id = id ) print('\nDeleting document:') @@ -114,7 +114,7 @@ print(response) # Delete the index. response = client.indices.delete( - index=index_name + index = index_name ) print('\nDeleting index:') From 30facfe628124b75cf17d44c90e946a20b1ce8ca Mon Sep 17 00:00:00 2001 From: aetter Date: Thu, 19 Aug 2021 14:38:30 -0700 Subject: [PATCH 005/229] Typo --- _clients/python.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_clients/python.md b/_clients/python.md index 0a44cfa9..879513ce 100644 --- a/_clients/python.md +++ b/_clients/python.md @@ -6,7 +6,7 @@ nav_order: 70 # Python client -The OpenSearch Python client provides a more natural syntax for interacting with your cluster. Rather than sending HTTP requests raw JSON bodies to a given URL, you can create an OpenSearch client for your cluster and call the client's built-in functions. +The OpenSearch Python client provides a more natural syntax for interacting with your cluster. Rather than sending HTTP requests with raw JSON bodies to a given URL, you can create an OpenSearch client for your cluster and call the client's built-in functions. ## Setup From 12ce7e5fea5c7c5499da834b36b2617afff79c30 Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Wed, 25 Aug 2021 11:05:24 -0700 Subject: [PATCH 006/229] rough draft --- _clients/go.md | 93 ++++++++++++++++++++++++++++++++ _clients/java-rest-high-level.md | 2 +- 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 _clients/go.md diff --git a/_clients/go.md b/_clients/go.md new file mode 100644 index 00000000..4273d8b9 --- /dev/null +++ b/_clients/go.md @@ -0,0 +1,93 @@ +--- +layout: default +title: Go client +nav_order: 80 +--- + +# Go client + +<> + +## Setup + +To add the client to your project, import it like any other module: + +```go +go mod init +go get github.com/opensearch-project/opensearch-go +``` + +### Sample response + +```go +go: downloading github.com/opensearch-project/opensearch-go v0.0.0-20210823214927-3567f157dece +go get: added github.com/opensearch-project/opensearch-go v0.0.0-20210823214927-3567f157dece +``` + +## Sample code + +Create a new file called main.go: + + +```go +package main + +import ( + "crypto/tls" + "fmt" + "github.com/opensearch-project/opensearch-go" + "log" + "net/http" +) + +func main () { + // Instantiate a new OpenSearch client object instance + + client, err := opensearch.NewClient(opensearch.Config{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + Addresses: []string{"https://localhost:9200"}, + Username: "admin", // For testing only. Don't store credentials in code. + Password: "admin", + }) + +// ca_certs_path = '/full/path/to/root-ca.pem' # Provide a CA bundle if you use intermediate CAs with your root CA. Optional client certificates if you don't want to use HTTP basic authentication. +// client_cert_path = '/full/path/to/client.pem' +// client_key_path = '/full/path/to/client-key.pem' + + if err != nil { + fmt.Println("cannot initialize", err) + } + +// Have the client instance return a response + + log.Println(client.Info()) +} +``` + +```go +$ go run main.go + +2021/08/24 23:56:05 [200 OK] { + "name" : "c825aab1d9cc", + "cluster_name" : "docker-cluster", + "cluster_uuid" : "jsIDR0FTR5qXZr4XkZ-GlA", + "version" : { + "distribution" : "opensearch", + "number" : "1.0.0", + "build_type" : "tar", + "build_hash" : "34550c5b17124ddc59458ef774f6b43a086522e3", + "build_date" : "2021-07-02T23:22:21.383695Z", + "build_snapshot" : false, + "lucene_version" : "8.8.2", + "minimum_wire_compatibility_version" : "6.8.0", + "minimum_index_compatibility_version" : "6.0.0-beta1" + }, + "tagline" : "The OpenSearch Project: https://opensearch.org/" +} + +``` + +For all APIs - +https://github.com/opensearch-project/opensearch-go/tree/main/opensearchapi diff --git a/_clients/java-rest-high-level.md b/_clients/java-rest-high-level.md index a9ed5945..def97086 100644 --- a/_clients/java-rest-high-level.md +++ b/_clients/java-rest-high-level.md @@ -1,7 +1,7 @@ --- layout: default title: Java high-level REST client -nav_order: 97 +nav_order: 60 --- # Java high-level REST client From ac9acb3c62287e1403fc0ad7c29cf28c1ddaee8e Mon Sep 17 00:00:00 2001 From: aetter Date: Thu, 26 Aug 2021 09:15:50 -0700 Subject: [PATCH 007/229] Adds some statements around compatibility Should (hopefully) apply to all clients. --- _clients/index.md | 14 +++++++++++++- _clients/python.md | 2 ++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/_clients/index.md b/_clients/index.md index bdf2bf05..3880a3bb 100644 --- a/_clients/index.md +++ b/_clients/index.md @@ -9,6 +9,18 @@ redirect_from: # OpenSearch client compatibility +OpenSearch provides clients for several popular programming languages, with more coming. In general, clients are compatible with clusters running the same major version of OpenSearch (`major.minor.patch`). + +For example, a 1.0.0 client works with an OpenSearch 1.1.0 cluster, but might not support any non-breaking API changes in OpenSearch 1.1.0. A 1.2.0 client works with the same cluster, but might allow you to pass unsupported options in certain functions. We recommend using the same version for both, but if your tests pass after a cluster upgrade, you don't necessarily need to upgrade your clients immediately. + +* [OpenSearch Java client]({{site.url}}{{site.baseurl}}/clients/java/) +* [OpenSearch Python client]({{site.url}}{{site.baseurl}}/clients/python/) +* [OpenSearch JavaScript (Node.js) client]({{site.url}}{{site.baseurl}}/clients/nodejs/) +* [OpenSearch Go client]({{site.url}}{{site.baseurl}}/clients/golang/) + + +## Legacy clients + Most clients that work with Elasticsearch OSS 7.10.2 *should* work with OpenSearch, but the latest versions of those clients might include license or version checks that artificially break compatibility. This page includes recommendations around which versions of those clients to use for best compatibility with OpenSearch. Client | Recommended version @@ -18,7 +30,7 @@ Client | Recommended version [Python Elasticsearch client](https://pypi.org/project/elasticsearch/7.13.4/) | 7.13.4 [Elasticsearch Node.js client](https://www.npmjs.com/package/@elastic/elasticsearch/v/7.13.0) | 7.13.0 -Clients exist for a wide variety of languages, so if you test a client and verify that it works, please [submit a PR](https://github.com/opensearch-project/documentation-website/pulls) and add it to this table. +If you test a legacy client and verify that it works, please [submit a PR](https://github.com/opensearch-project/documentation-website/pulls) and add it to this table. {% comment %} diff --git a/_clients/python.md b/_clients/python.md index 879513ce..f2abffe5 100644 --- a/_clients/python.md +++ b/_clients/python.md @@ -23,6 +23,8 @@ Then import it like any other module: from opensearch import OpenSearch ``` +If you prefer to add the client manually or just want to examine the source code, see [opensearch-py on GitHub](https://github.com/opensearch-project/opensearch-py). + ## Sample code From 9a2e6ed0284c65bf0ac2620feba7d8dc4d15c3b4 Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Thu, 26 Aug 2021 11:33:21 -0700 Subject: [PATCH 008/229] spacing --- _clients/go.md | 132 ++++++++++++++++++++++++++++++------------------- 1 file changed, 82 insertions(+), 50 deletions(-) diff --git a/_clients/go.md b/_clients/go.md index 4273d8b9..d9bc751a 100644 --- a/_clients/go.md +++ b/_clients/go.md @@ -6,7 +6,8 @@ nav_order: 80 # Go client -<> +The OpenSearch Go client lets you programmatically interact with data in your OpenSearch cluster as part of your Go application. + ## Setup @@ -17,77 +18,108 @@ go mod init go get github.com/opensearch-project/opensearch-go ``` -### Sample response - -```go -go: downloading github.com/opensearch-project/opensearch-go v0.0.0-20210823214927-3567f157dece -go get: added github.com/opensearch-project/opensearch-go v0.0.0-20210823214927-3567f157dece -``` - ## Sample code -Create a new file called main.go: - - ```go package main import ( + "context" "crypto/tls" "fmt" - "github.com/opensearch-project/opensearch-go" - "log" + opensearch "github.com/opensearch-project/opensearch-go" + opensearchapi "github.com/opensearch-project/opensearch-go/opensearchapi" "net/http" + "strings" ) -func main () { - // Instantiate a new OpenSearch client object instance +const IndexName = "go-test-index1" +func main() { + + // Initialize the client with SSL/TLS enabled. client, err := opensearch.NewClient(opensearch.Config{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, Addresses: []string{"https://localhost:9200"}, - Username: "admin", // For testing only. Don't store credentials in code. - Password: "admin", + Username: "admin", // For testing only. Don't store credentials in code. + Password: "admin", }) - -// ca_certs_path = '/full/path/to/root-ca.pem' # Provide a CA bundle if you use intermediate CAs with your root CA. Optional client certificates if you don't want to use HTTP basic authentication. -// client_cert_path = '/full/path/to/client.pem' -// client_key_path = '/full/path/to/client-key.pem' - if err != nil { fmt.Println("cannot initialize", err) } -// Have the client instance return a response - + // Print OpenSearch version information. log.Println(client.Info()) + + // Define a mapping. + mapping := strings.NewReader(`{ + 'settings': { + 'index': { + 'number_of_shards': 4 + } + } + }`) + + // Create an index with non-default settings. + res := opensearchapi.CreateRequest{ + Index: IndexName, + Body: mapping, + } + fmt.Println("creating index", res) + + // Add a document to the index. + document := strings.NewReader(`{ + "title": "Moneyball", + "director": "Bennett Miller", + "year": "2011" + }`) + + docId := "1" + req := opensearchapi.IndexRequest{ + Index: IndexName, + DocumentID: docId, + Body: document, + } + insertResponse, err := req.Do(context.Background(), client) + fmt.Println(insertResponse) + + // Search for the document. + content := strings.NewReader(`{ + "size": 5, + "query": { + "multi_match": { + "query": "miller", + "fields": ["title^2", "director"] + } + } + }`) + + search := opensearchapi.SearchRequest{ + Body: content, + } + + searchResponse, err := search.Do(context.Background(), client) + fmt.Println(searchResponse) + + // Delete the document. + delete := opensearchapi.DeleteRequest{ + Index: IndexName, + DocumentID: docId, + } + + deleteResponse, err := delete.Do(context.Background(), client) + fmt.Println("deleting document") + fmt.Println(deleteResponse) + + // Delete the index. + deleteIndex := opensearchapi.IndicesDeleteRequest{ + Index: []string{IndexName}, + } + + deleteIndexResponse, err := deleteIndex.Do(context.Background(), client) + fmt.Println("deleting index") + fmt.Println(deleteIndexResponse) } ``` - -```go -$ go run main.go - -2021/08/24 23:56:05 [200 OK] { - "name" : "c825aab1d9cc", - "cluster_name" : "docker-cluster", - "cluster_uuid" : "jsIDR0FTR5qXZr4XkZ-GlA", - "version" : { - "distribution" : "opensearch", - "number" : "1.0.0", - "build_type" : "tar", - "build_hash" : "34550c5b17124ddc59458ef774f6b43a086522e3", - "build_date" : "2021-07-02T23:22:21.383695Z", - "build_snapshot" : false, - "lucene_version" : "8.8.2", - "minimum_wire_compatibility_version" : "6.8.0", - "minimum_index_compatibility_version" : "6.0.0-beta1" - }, - "tagline" : "The OpenSearch Project: https://opensearch.org/" -} - -``` - -For all APIs - -https://github.com/opensearch-project/opensearch-go/tree/main/opensearchapi From bfc56f2f7fbf9cf97910483bddb9ca0286b640bc Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Thu, 26 Aug 2021 11:39:32 -0700 Subject: [PATCH 009/229] incorporated feedback --- _clients/go.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/_clients/go.md b/_clients/go.md index d9bc751a..37c89e94 100644 --- a/_clients/go.md +++ b/_clients/go.md @@ -11,10 +11,15 @@ The OpenSearch Go client lets you programmatically interact with data in your Op ## Setup -To add the client to your project, import it like any other module: +If you're creating a new project: ```go go mod init +``` + +To add the client to your project, import it like any other module: + +```go go get github.com/opensearch-project/opensearch-go ``` From 1c4f81eb53185387b27afdd961b32739475a9478 Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Thu, 26 Aug 2021 11:41:09 -0700 Subject: [PATCH 010/229] minor fix --- _clients/go.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_clients/go.md b/_clients/go.md index 37c89e94..e77fc0dd 100644 --- a/_clients/go.md +++ b/_clients/go.md @@ -55,8 +55,8 @@ func main() { fmt.Println("cannot initialize", err) } - // Print OpenSearch version information. - log.Println(client.Info()) + // Print OpenSearch version information on console. + fmt.Println(client.Info()) // Define a mapping. mapping := strings.NewReader(`{ From c9728698934f57a36278d85314ab647407c06f90 Mon Sep 17 00:00:00 2001 From: aetter Date: Thu, 26 Aug 2021 11:50:46 -0700 Subject: [PATCH 011/229] Link to pip, because why not --- _clients/python.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_clients/python.md b/_clients/python.md index f2abffe5..cdc7dc29 100644 --- a/_clients/python.md +++ b/_clients/python.md @@ -11,7 +11,7 @@ The OpenSearch Python client provides a more natural syntax for interacting with ## Setup -To add the client to your project, install it using pip: +To add the client to your project, install it using [pip](https://pip.pypa.io/): ```bash pip install From cea3ba7ce9450617dac23521bd1f2277931d89dd Mon Sep 17 00:00:00 2001 From: Liz Snyder Date: Thu, 26 Aug 2021 12:44:58 -0700 Subject: [PATCH 012/229] Getting started docs for javascript --- _clients/javascript.md | 141 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 _clients/javascript.md diff --git a/_clients/javascript.md b/_clients/javascript.md new file mode 100644 index 00000000..60c1cba1 --- /dev/null +++ b/_clients/javascript.md @@ -0,0 +1,141 @@ +--- +layout: default +title: Javascript client +nav_order: 80 +--- + +# Javascript client + +The OpenSearch Javascript client provides a safer and easier way to interact with your OpenSearch cluster. Rather than using OpenSearch from the browser and potentially exposing your data to the public, you can build an OpenSearch client that takes care of sending requests to your cluster. + +The client contains a library of APIs that let you perform different operations on your cluster and return a standard response body. The example here demonstrates some basic operations like creating an index, adding documents, and searching your data. + + +## Setup + +To add the client to your project, install it from npm: + +```bash +npm install @opensearch/opensearch +``` + +If you prefer to add the client manually or just want to examine the source code, see [opensearch-js](https://github.com/opensearch-project/opensearch-js) on GitHub. + +Then require the client: + +```javascript +const { Client } = require('@opensearch/opensearch') +``` + + +## Sample code + +```javascript +'use strict'; + +var host = 'localhost'; +var protocol = 'https'; +var port = 9200; +var auth = 'admin:admin'; // For testing only. Don't store credentials in code. +var ca_certs_path = '/full/path/to/root-ca.pem'; + +// Optional client certificates if you don't want to use HTTP basic authentication. +// var client_cert_path = '/full/path/to/client.pem' +// var client_key_path = '/full/path/to/client-key.pem' + +// Create a client with SSL/TLS enabled. +var { Client } = require('@opensearch/opensearch'); +var fs = require('fs'); +var client = new Client({ + node: protocol + '://' + auth + '@' + host + ':' + port, + ssl: { + ca: fs.readFileSync(ca_certs_path), + // cert: fs.readFileSync(client_cert_path), + // key: fs.readFileSync(client_key_path) + } +}) + +async function search() { + + // Create an index with non-default settings. + var index_name = 'books' + var settings = { + 'settings': { + 'index': { + 'number_of_shards': 4, + 'number_of_replicas': 3 + } + } + } + + var response = await client.indices.create({ + index: index_name, + body: settings + }) + + console.log('Creating index:') + console.log(response.body) + + // Add a document to the index. + var document = { + 'title': 'The Outsider', + 'author': 'Stephen King', + 'year': '2018', + 'genre': 'Crime fiction' + } + + var id = '1' + + var response = await client.index({ + id: id, + index: index_name, + body: document + }) + + console.log('Adding document:') + console.log(response.body) + + // Force an index refresh so the subsequent search returns results. + client.indices.refresh({ + index: index_name + }) + + // Search for the document. + var query = { + 'query': { + 'match': { + 'title': { + 'query': 'The Outsider' + } + } + } + } + + var response = await client.search({ + index: index_name, + body: query + }) + + console.log('Search results:') + console.log(response.body.hits) + + // Delete the document. + var response = await client.delete({ + index: index_name, + id: id + }) + + console.log('Deleting document:') + console.log(response.body) + + // Delete the index. + var response = await client.indices.delete({ + index: index_name + }) + + console.log('Deleting index:') + console.log(response.body) +} + +search().catch(console.log) +``` From 4d39000cd3daa41336c031fbdf26566b2b8c25d1 Mon Sep 17 00:00:00 2001 From: Liz Snyder Date: Thu, 26 Aug 2021 12:51:04 -0700 Subject: [PATCH 013/229] Add command to install specific version --- _clients/javascript.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/_clients/javascript.md b/_clients/javascript.md index 60c1cba1..42a00f67 100644 --- a/_clients/javascript.md +++ b/_clients/javascript.md @@ -19,6 +19,12 @@ To add the client to your project, install it from npm: npm install @opensearch/opensearch ``` +To install a specific major version of the client, run the following command: + +```bash +npm install @opensearch/opensearch@ +``` + If you prefer to add the client manually or just want to examine the source code, see [opensearch-js](https://github.com/opensearch-project/opensearch-js) on GitHub. Then require the client: From 51d359ec4075a41e2bdb14567ea6fb1113f621fa Mon Sep 17 00:00:00 2001 From: Liz Snyder Date: Thu, 26 Aug 2021 13:34:19 -0700 Subject: [PATCH 014/229] Remove index refresh --- _clients/javascript.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/_clients/javascript.md b/_clients/javascript.md index 42a00f67..f6490863 100644 --- a/_clients/javascript.md +++ b/_clients/javascript.md @@ -95,17 +95,13 @@ async function search() { var response = await client.index({ id: id, index: index_name, - body: document + body: document, + refresh: true }) console.log('Adding document:') console.log(response.body) - // Force an index refresh so the subsequent search returns results. - client.indices.refresh({ - index: index_name - }) - // Search for the document. var query = { 'query': { From e210d7d217586d64f3421e80d9c4af892237b74c Mon Sep 17 00:00:00 2001 From: Liz Snyder Date: Thu, 26 Aug 2021 13:36:12 -0700 Subject: [PATCH 015/229] Fix nav order to not conflict with Go --- _clients/javascript.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_clients/javascript.md b/_clients/javascript.md index f6490863..2c4946f4 100644 --- a/_clients/javascript.md +++ b/_clients/javascript.md @@ -1,7 +1,7 @@ --- layout: default title: Javascript client -nav_order: 80 +nav_order: 90 --- # Javascript client From 18d389187943d1a3002ac916a5f6741b7573c148 Mon Sep 17 00:00:00 2001 From: Liz Snyder Date: Thu, 26 Aug 2021 15:23:06 -0700 Subject: [PATCH 016/229] Specify how to turn off cert verification --- _clients/javascript.md | 1 + 1 file changed, 1 insertion(+) diff --git a/_clients/javascript.md b/_clients/javascript.md index 2c4946f4..55134c00 100644 --- a/_clients/javascript.md +++ b/_clients/javascript.md @@ -56,6 +56,7 @@ var client = new Client({ node: protocol + '://' + auth + '@' + host + ':' + port, ssl: { ca: fs.readFileSync(ca_certs_path), + // You can turn off certificate verification (rejectUnauthorized: false) if you're using self-signed certificates with a hostname mismatch. // cert: fs.readFileSync(client_cert_path), // key: fs.readFileSync(client_key_path) } From fbc0447bcdad8e2a83caa74153d3a46d3a5b280d Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Fri, 27 Aug 2021 13:20:59 -0700 Subject: [PATCH 017/229] added more context --- _clients/go.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/_clients/go.md b/_clients/go.md index e77fc0dd..06b6e533 100644 --- a/_clients/go.md +++ b/_clients/go.md @@ -25,6 +25,8 @@ go get github.com/opensearch-project/opensearch-go ## Sample code +This sample code creates a client, adds an index with non-default settings, inserts a document, searches for the document, deletes the document, and finally deletes the index: + ```go package main From 7aed3bfbf1a1af94a0c21795bbc2165d9df14c09 Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Fri, 27 Aug 2021 14:00:26 -0700 Subject: [PATCH 018/229] updated code to check error message --- _clients/go.md | 173 ++++++++++++++++++++++++++----------------------- 1 file changed, 93 insertions(+), 80 deletions(-) diff --git a/_clients/go.md b/_clients/go.md index 06b6e533..6db260ac 100644 --- a/_clients/go.md +++ b/_clients/go.md @@ -29,104 +29,117 @@ This sample code creates a client, adds an index with non-default settings, inse ```go package main - import ( - "context" - "crypto/tls" - "fmt" - opensearch "github.com/opensearch-project/opensearch-go" - opensearchapi "github.com/opensearch-project/opensearch-go/opensearchapi" - "net/http" - "strings" + "os" + "context" + "crypto/tls" + "fmt" + opensearch "github.com/opensearch-project/opensearch-go" + opensearchapi "github.com/opensearch-project/opensearch-go/opensearchapi" + "net/http" + "strings" ) - const IndexName = "go-test-index1" - func main() { + // Initialize the client with SSL/TLS enabled. + client, err := opensearch.NewClient(opensearch.Config{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + Addresses: []string{"https://localhost:9200"}, + Username: "admin", // For testing only. Don't store credentials in code. + Password: "admin", + }) + if err != nil { + fmt.Println("cannot initialize", err) + os.Exit(1) + } - // Initialize the client with SSL/TLS enabled. - client, err := opensearch.NewClient(opensearch.Config{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, - Addresses: []string{"https://localhost:9200"}, - Username: "admin", // For testing only. Don't store credentials in code. - Password: "admin", - }) - if err != nil { - fmt.Println("cannot initialize", err) - } + // Print OpenSearch version information on console. + fmt.Println(client.Info()) - // Print OpenSearch version information on console. - fmt.Println(client.Info()) - - // Define a mapping. - mapping := strings.NewReader(`{ - 'settings': { - 'index': { + // Define index mapping. + mapping := strings.NewReader(`{ + 'settings': { + 'index': { 'number_of_shards': 4 } } - }`) + }`) - // Create an index with non-default settings. - res := opensearchapi.CreateRequest{ - Index: IndexName, - Body: mapping, - } - fmt.Println("creating index", res) + // Create an index with non-default settings. + res := opensearchapi.CreateRequest{ + Index: IndexName, + Body: mapping, + } + fmt.Println("creating index", res) - // Add a document to the index. - document := strings.NewReader(`{ - "title": "Moneyball", - "director": "Bennett Miller", - "year": "2011" - }`) + // Add a document to the index. + document := strings.NewReader(`{ + "title": "Moneyball", + "director": "Bennett Miller", + "year": "2011" + }`) - docId := "1" - req := opensearchapi.IndexRequest{ - Index: IndexName, - DocumentID: docId, - Body: document, - } - insertResponse, err := req.Do(context.Background(), client) - fmt.Println(insertResponse) + docId := "1" + req := opensearchapi.IndexRequest{ + Index: IndexName, + DocumentID: docId, + Body: document, + } + insertResponse, err := req.Do(context.Background(), client) + if err != nil { + fmt.Println("failed to insert document ", err) + os.Exit(1) + } + fmt.Println(insertResponse) - // Search for the document. - content := strings.NewReader(`{ - "size": 5, - "query": { - "multi_match": { - "query": "miller", - "fields": ["title^2", "director"] - } - } - }`) + // Search for the document. + content := strings.NewReader(`{ + "size": 5, + "query": { + "multi_match": { + "query": "miller", + "fields": ["title^2", "director"] + } + } + }`) - search := opensearchapi.SearchRequest{ - Body: content, - } + search := opensearchapi.SearchRequest{ + Body: content, + } - searchResponse, err := search.Do(context.Background(), client) - fmt.Println(searchResponse) + searchResponse, err := search.Do(context.Background(), client) + if err != nil { + fmt.Println("failed to search document ", err) + os.Exit(1) + } + fmt.Println(searchResponse) - // Delete the document. - delete := opensearchapi.DeleteRequest{ - Index: IndexName, - DocumentID: docId, - } + // Delete the document. + delete := opensearchapi.DeleteRequest{ + Index: IndexName, + DocumentID: docId, + } - deleteResponse, err := delete.Do(context.Background(), client) - fmt.Println("deleting document") - fmt.Println(deleteResponse) + deleteResponse, err := delete.Do(context.Background(), client) + if err != nil { + fmt.Println("failed to delete document ", err) + os.Exit(1) + } + fmt.Println("deleting document") + fmt.Println(deleteResponse) - // Delete the index. - deleteIndex := opensearchapi.IndicesDeleteRequest{ - Index: []string{IndexName}, - } + // Delete previously created index. + deleteIndex := opensearchapi.IndicesDeleteRequest{ + Index: []string{IndexName}, + } - deleteIndexResponse, err := deleteIndex.Do(context.Background(), client) - fmt.Println("deleting index") - fmt.Println(deleteIndexResponse) + deleteIndexResponse, err := deleteIndex.Do(context.Background(), client) + if err != nil { + fmt.Println("failed to delete index ", err) + os.Exit(1) + } + fmt.Println("deleting index", deleteIndexResponse) } ``` From 0b9f1973581ecfa81065ccaf06c9cacf1026b403 Mon Sep 17 00:00:00 2001 From: keithhc2 Date: Fri, 10 Sep 2021 11:57:30 -0700 Subject: [PATCH 019/229] Added security to ISM --- _im-plugin/ism/api.md | 2 +- _im-plugin/ism/security.md | 40 +++++++++++++++++++ .../access-control/users-roles.md | 2 + 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 _im-plugin/ism/security.md diff --git a/_im-plugin/ism/api.md b/_im-plugin/ism/api.md index af139695..3f7e0d1c 100644 --- a/_im-plugin/ism/api.md +++ b/_im-plugin/ism/api.md @@ -2,7 +2,7 @@ layout: default title: ISM API parent: Index State Management -nav_order: 5 +nav_order: 20 --- # ISM API diff --git a/_im-plugin/ism/security.md b/_im-plugin/ism/security.md new file mode 100644 index 00000000..b996e319 --- /dev/null +++ b/_im-plugin/ism/security.md @@ -0,0 +1,40 @@ +--- +layout: default +title: ISM security +nav_order: 10 +parent: Index State Management +has_children: false +--- + +# ISM security + +Using the security plugin with index state management lets you limit non-admin users to certain actions. For example, you might want to set up your security such that a group of users can only read ISM policies, while others can create, delete, or change policies. + +All index state management data are protected as system indices, and only a super admin or an admin with a Transport Layer Security (TLS) certificate can access system indices. For more information, see [System indices]({{site.url}}{{site.baseurl}}/security-plugin/configuration/system-indices). + +## Basic permissions + +The security plugin comes with two built-in roles that cover most ISM use cases: `ism_read_access` and `ism_full_access`. For descriptions of each, see [Predefined roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/users-roles#predefined-roles). + +With security enabled, users not only need the correct index management permissions, but they also need permissions to relevant indices to execute [REST API]({{site.url}}{{site.baseurl}}/im-plugin/ism/api) calls. For example, if a user wants to use the REST API to attach a policy to an index named `system-logs`, they would need the necessary REST API permissions as well as access to `system-logs`. + +## (Advanced) Limit access by backend role + +You can use backend roles to configure fine-grained access to ISM policies and actions. For example, users of different departments in an organization might view different policies depending on what roles and permissions they are assigned. + +First, ensure your users have the appropriate [backend roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/index/). Backend roles usually come from an [LDAP server]({{site.url}}{{site.baseurl}}/security-plugin/configuration/ldap/) or [SAML provider]({{site.url}}{{site.baseurl}}/security-plugin/configuration/saml/). However, if you use the internal user database, you can use the REST API to [add them manually]({{site.url}}{{site.baseurl}}/security-plugin/access-control/api#create-user). + +Use the REST API to enable the following setting: + +```json +PUT _cluster/settings +{ + "transient": { + "plugins.index_management.filter_by_backend_roles": "true" + } +} +``` + +With security enabled, only users who share at least one backend role can see and execute the policies and actions relevant to their roles. + +For example, consider a scenario with three users: `John` and `Jill`, who have the backend role `helpdesk_staff`, and `Jane`, who has the backend role `phone_operator`. `John` wants to create a policy that performs a rollup job on an index named `airline_data`, so `John` would need a backend role that has permissions to access that index, create relevant policies, and execute relevant actions, and `Jill` would be able to access the same index, policy, and job. However, `Jane` cannot access or edit those resources or actions. diff --git a/_security-plugin/access-control/users-roles.md b/_security-plugin/access-control/users-roles.md index 445e1e63..1422d972 100644 --- a/_security-plugin/access-control/users-roles.md +++ b/_security-plugin/access-control/users-roles.md @@ -121,6 +121,8 @@ Role | Description `reports_full_access` | Grants full permissions to reports. `asynchronous_search_full_access` | Grants full permissions to all asynchronous search actions. `asynchronous_search_read_access` | Grants permissions to view asynchronous searches, but not to submit, modify, or delete async searches. +`ism_read_access` | Grants users permissions to view policies and current index states, but they are unable to create, add, update, or remove policies. +`ism_full_access` | Grants full permissions to all ISM actions. For more detailed summaries of the permissions for each role, reference their action groups against the descriptions in [Default action groups]({{site.url}}{{site.baseurl}}/security-plugin/access-control/default-action-groups/). From 50f3fa51d84c99f38580da1bb91523f63874a798 Mon Sep 17 00:00:00 2001 From: aetter Date: Fri, 10 Sep 2021 15:01:43 -0700 Subject: [PATCH 020/229] Update python.md --- _clients/python.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_clients/python.md b/_clients/python.md index cdc7dc29..bcb1ff93 100644 --- a/_clients/python.md +++ b/_clients/python.md @@ -14,7 +14,7 @@ The OpenSearch Python client provides a more natural syntax for interacting with To add the client to your project, install it using [pip](https://pip.pypa.io/): ```bash -pip install +pip install opensearch ``` Then import it like any other module: From a69440b2626e495f438468e1d064ed4821ccee40 Mon Sep 17 00:00:00 2001 From: keithhc2 Date: Wed, 15 Sep 2021 13:47:04 -0700 Subject: [PATCH 021/229] Addressed comments --- _im-plugin/ism/security.md | 40 ------------------ _im-plugin/refresh-analyzer/index.md | 2 +- _im-plugin/security.md | 41 +++++++++++++++++++ .../access-control/users-roles.md | 3 +- 4 files changed, 43 insertions(+), 43 deletions(-) delete mode 100644 _im-plugin/ism/security.md create mode 100644 _im-plugin/security.md diff --git a/_im-plugin/ism/security.md b/_im-plugin/ism/security.md deleted file mode 100644 index b996e319..00000000 --- a/_im-plugin/ism/security.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -layout: default -title: ISM security -nav_order: 10 -parent: Index State Management -has_children: false ---- - -# ISM security - -Using the security plugin with index state management lets you limit non-admin users to certain actions. For example, you might want to set up your security such that a group of users can only read ISM policies, while others can create, delete, or change policies. - -All index state management data are protected as system indices, and only a super admin or an admin with a Transport Layer Security (TLS) certificate can access system indices. For more information, see [System indices]({{site.url}}{{site.baseurl}}/security-plugin/configuration/system-indices). - -## Basic permissions - -The security plugin comes with two built-in roles that cover most ISM use cases: `ism_read_access` and `ism_full_access`. For descriptions of each, see [Predefined roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/users-roles#predefined-roles). - -With security enabled, users not only need the correct index management permissions, but they also need permissions to relevant indices to execute [REST API]({{site.url}}{{site.baseurl}}/im-plugin/ism/api) calls. For example, if a user wants to use the REST API to attach a policy to an index named `system-logs`, they would need the necessary REST API permissions as well as access to `system-logs`. - -## (Advanced) Limit access by backend role - -You can use backend roles to configure fine-grained access to ISM policies and actions. For example, users of different departments in an organization might view different policies depending on what roles and permissions they are assigned. - -First, ensure your users have the appropriate [backend roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/index/). Backend roles usually come from an [LDAP server]({{site.url}}{{site.baseurl}}/security-plugin/configuration/ldap/) or [SAML provider]({{site.url}}{{site.baseurl}}/security-plugin/configuration/saml/). However, if you use the internal user database, you can use the REST API to [add them manually]({{site.url}}{{site.baseurl}}/security-plugin/access-control/api#create-user). - -Use the REST API to enable the following setting: - -```json -PUT _cluster/settings -{ - "transient": { - "plugins.index_management.filter_by_backend_roles": "true" - } -} -``` - -With security enabled, only users who share at least one backend role can see and execute the policies and actions relevant to their roles. - -For example, consider a scenario with three users: `John` and `Jill`, who have the backend role `helpdesk_staff`, and `Jane`, who has the backend role `phone_operator`. `John` wants to create a policy that performs a rollup job on an index named `airline_data`, so `John` would need a backend role that has permissions to access that index, create relevant policies, and execute relevant actions, and `Jill` would be able to access the same index, policy, and job. However, `Jane` cannot access or edit those resources or actions. diff --git a/_im-plugin/refresh-analyzer/index.md b/_im-plugin/refresh-analyzer/index.md index d9beb9bb..641d3484 100644 --- a/_im-plugin/refresh-analyzer/index.md +++ b/_im-plugin/refresh-analyzer/index.md @@ -1,7 +1,7 @@ --- layout: default title: Refresh search analyzer -nav_order: 40 +nav_order: 50 has_children: false redirect_from: /im-plugin/refresh-analyzer/ has_toc: false diff --git a/_im-plugin/security.md b/_im-plugin/security.md new file mode 100644 index 00000000..2fa6d456 --- /dev/null +++ b/_im-plugin/security.md @@ -0,0 +1,41 @@ +--- +layout: default +title: Index management security +nav_order: 40 +has_children: false +--- + +# Index management security + +Using the security plugin with index management lets you limit non-admin users to certain actions. For example, you might want to set up your security such that a group of users can only read ISM policies, while others can create, delete, or change policies. + +All index management data are protected as system indices, and only a super admin or an admin with a Transport Layer Security (TLS) certificate can access system indices. For more information, see [System indices]({{site.url}}{{site.baseurl}}/security-plugin/configuration/system-indices). + +## Basic permissions + +The security plugin comes with one role that offers full access to index management: `index_management_full_access`. For a description of the role's permissions, see [Predefined roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/users-roles#predefined-roles). + +With security enabled, users not only need the correct index management permissions, but they also need permissions to execute actions to involved indices. For example, if a user wants to use the REST API to attach a policy that executes a transform job to an index named `system-logs`, they would need the permissions to attach a policy and execute a transform job, as well as access to `system-logs`. + +Finally, with the exceptions of Create Policy, Get Policy, and Delete Policy, users also need the `indices:admin/opensearch/ism/managedindex` permission to execute [ISM APIs]({{site.url}}{{site.baseurl}}/im-plugin/ism/api). + +## (Advanced) Limit access by backend role + +You can use backend roles to configure fine-grained access to index management policies and actions. For example, users of different departments in an organization might view different policies depending on what roles and permissions they are assigned. + +First, ensure your users have the appropriate [backend roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/index/). Backend roles usually come from an [LDAP server]({{site.url}}{{site.baseurl}}/security-plugin/configuration/ldap/) or [SAML provider]({{site.url}}{{site.baseurl}}/security-plugin/configuration/saml/). However, if you use the internal user database, you can use the REST API to [add them manually]({{site.url}}{{site.baseurl}}/security-plugin/access-control/api#create-user). + +Use the REST API to enable the following setting: + +```json +PUT _cluster/settings +{ + "transient": { + "plugins.index_management.filter_by_backend_roles": "true" + } +} +``` + +With security enabled, only users who share at least one backend role can see and execute the policies and actions relevant to their roles. + +For example, consider a scenario with three users: `John` and `Jill`, who have the backend role `helpdesk_staff`, and `Jane`, who has the backend role `phone_operator`. `John` wants to create a policy that performs a rollup job on an index named `airline_data`, so `John` would need a backend role that has permissions to access that index, create relevant policies, and execute relevant actions, and `Jill` would be able to access the same index, policy, and job. However, `Jane` cannot access or edit those resources or actions. diff --git a/_security-plugin/access-control/users-roles.md b/_security-plugin/access-control/users-roles.md index 1422d972..86424304 100644 --- a/_security-plugin/access-control/users-roles.md +++ b/_security-plugin/access-control/users-roles.md @@ -121,8 +121,7 @@ Role | Description `reports_full_access` | Grants full permissions to reports. `asynchronous_search_full_access` | Grants full permissions to all asynchronous search actions. `asynchronous_search_read_access` | Grants permissions to view asynchronous searches, but not to submit, modify, or delete async searches. -`ism_read_access` | Grants users permissions to view policies and current index states, but they are unable to create, add, update, or remove policies. -`ism_full_access` | Grants full permissions to all ISM actions. +`index_management_full_access` | Grants full permissions to all index management actions, including ISM, transforms, and rollups. For more detailed summaries of the permissions for each role, reference their action groups against the descriptions in [Default action groups]({{site.url}}{{site.baseurl}}/security-plugin/access-control/default-action-groups/). From 538703d4227378c173565b5f30e5c56125eb9c3e Mon Sep 17 00:00:00 2001 From: keithhc2 Date: Fri, 17 Sep 2021 14:58:07 -0700 Subject: [PATCH 022/229] Changed transform to rollup --- _im-plugin/security.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_im-plugin/security.md b/_im-plugin/security.md index 2fa6d456..d5d48ac6 100644 --- a/_im-plugin/security.md +++ b/_im-plugin/security.md @@ -15,7 +15,7 @@ All index management data are protected as system indices, and only a super admi The security plugin comes with one role that offers full access to index management: `index_management_full_access`. For a description of the role's permissions, see [Predefined roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/users-roles#predefined-roles). -With security enabled, users not only need the correct index management permissions, but they also need permissions to execute actions to involved indices. For example, if a user wants to use the REST API to attach a policy that executes a transform job to an index named `system-logs`, they would need the permissions to attach a policy and execute a transform job, as well as access to `system-logs`. +With security enabled, users not only need the correct index management permissions, but they also need permissions to execute actions to involved indices. For example, if a user wants to use the REST API to attach a policy that executes a rollup job to an index named `system-logs`, they would need the permissions to attach a policy and execute a rollup job, as well as access to `system-logs`. Finally, with the exceptions of Create Policy, Get Policy, and Delete Policy, users also need the `indices:admin/opensearch/ism/managedindex` permission to execute [ISM APIs]({{site.url}}{{site.baseurl}}/im-plugin/ism/api). From 9738b5c54b5ca1f7f78a0cb1bc1e540aaa462f2d Mon Sep 17 00:00:00 2001 From: keithhc2 Date: Fri, 17 Sep 2021 17:18:46 -0700 Subject: [PATCH 023/229] Finally adding new OpenSearch java client --- _clients/java-rest-high-level.md | 6 +- _clients/java.md | 101 +++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 3 deletions(-) create mode 100644 _clients/java.md diff --git a/_clients/java-rest-high-level.md b/_clients/java-rest-high-level.md index a9ed5945..5d418d3b 100644 --- a/_clients/java-rest-high-level.md +++ b/_clients/java-rest-high-level.md @@ -1,10 +1,10 @@ --- layout: default -title: Java high-level REST client +title: Elasticsearch OSS Java high-level REST client nav_order: 97 --- -# Java high-level REST client +# Elasticsearch OSS Java high-level REST client The Elasticsearch OSS Java high-level REST client allows you to interact with your OpenSearch clusters and indices through Java methods and data structures rather than HTTP methods and JSON. @@ -22,7 +22,7 @@ To start using the Elasticsearch OSS Java high-level REST client, ensure that yo ``` -You can now start your OpenSearch cluster. The 7.10.2 high-level REST client works with the 1.x versions of OpenSearch. +You can now start your OpenSearch cluster. The 7.10.2 Elasticsearch OSS high-level REST client works with the 1.x versions of OpenSearch. ## Sample code diff --git a/_clients/java.md b/_clients/java.md new file mode 100644 index 00000000..9d49a279 --- /dev/null +++ b/_clients/java.md @@ -0,0 +1,101 @@ +--- +layout: default +title: OpenSearch Java client +nav_order: 65 +--- + +# Java client + +The OpenSearch Java client allows you to interact with your OpenSearch clusters through Java methods and data structures rather than HTTP methods and raw JSON. For example, you can submit requests to your cluster using objects to create indices, add data to documents, or complete some other operation using the client's built-in methods. + +## Setup + +To start using the OpenSearch Java client, ensure that you have the following dependency in your project's `pom.xml` file: + +``` + + + +``` + +You can now start your OpenSearch cluster. + +## Sample code + +```java +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; +import org.opensearch.clients.base.RestClientTransport; +import org.opensearch.clients.base.Transport; +import org.opensearch.clients.json.jackson.JacksonJsonpMapper; +import org.opensearch.clients.opensearch.OpenSearchClient; +import org.opensearch.clients.opensearch._global.IndexRequest; +import org.opensearch.clients.opensearch._global.IndexResponse; +import org.opensearch.clients.opensearch._global.SearchResponse; +import org.opensearch.clients.opensearch.indices.*; +import org.opensearch.clients.opensearch.indices.put_settings.IndexSettingsBody; + +import java.io.IOException; + + +public class Main { + public static void main(String[] args) throws IOException{ + System.setProperty("javax.net.ssl.trustStore", "/full/path/to/keystore"); + System.setProperty("javax.net.ssl.trustStorePassword", password-to-keystore); + + //Only for demo purposes. Don't specify your credentials in code. + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials("admin", "admin")); + + //Initialize the client with SSL and TLS enabled + RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200, "https")). + setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + } + }).build(); + Transport transport = new RestClientTransport(restClient, new JacksonJsonpMapper()); + OpenSearchClient client = new OpenSearchClient(transport); + + //Create the index + String index = "sample-index"; + CreateRequest createIndexRequest = new CreateRequest.Builder().index(index).build(); + client.indices().create(createIndexRequest); + + //Add some settings to the index + IndexSettings indexSettings = new IndexSettings.Builder().autoExpandReplicas("0-all").build(); + IndexSettingsBody settingsBody = new IndexSettingsBody.Builder().settings(indexSettings).build(); + PutSettingsRequest putSettingsRequest = new PutSettingsRequest.Builder().index(index).value(settingsBody).build(); + client.indices().putSettings(putSettingsRequest); + + //Index some data + IndexData indexData = new IndexData("first_name", "Bruce"); + IndexRequest indexRequest = new IndexRequest.Builder().index(index).id("1").value(indexData).build(); + client.index(indexRequest); + + //Search for the document + SearchResponse searchResponse = client.search(s -> s.index(index), IndexData.class); + for (int i = 0; i< searchResponse.hits().hits().size(); i++) { + System.out.println(searchResponse.hits().hits().get(i).source()); + } + + //Delete the document + client.delete(b -> b.index(index).id("1")); + + // Delete the index + DeleteRequest deleteRequest = new DeleteRequest.Builder().index(index).build(); + DeleteResponse deleteResponse = client.indices().delete(deleteRequest); + + restClient.close(); + } +} + +``` From 9861e07d7cc57ae241ac22165fa56a00d705eddc Mon Sep 17 00:00:00 2001 From: Keith Chan <12404772+keithhc2@users.noreply.github.com> Date: Mon, 20 Sep 2021 13:18:21 -0700 Subject: [PATCH 024/229] Added quotes around keystore password --- _clients/java.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_clients/java.md b/_clients/java.md index 9d49a279..3823d0c4 100644 --- a/_clients/java.md +++ b/_clients/java.md @@ -47,7 +47,7 @@ import java.io.IOException; public class Main { public static void main(String[] args) throws IOException{ System.setProperty("javax.net.ssl.trustStore", "/full/path/to/keystore"); - System.setProperty("javax.net.ssl.trustStorePassword", password-to-keystore); + System.setProperty("javax.net.ssl.trustStorePassword", "password-to-keystore"); //Only for demo purposes. Don't specify your credentials in code. final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); From 05ba9198f60666f343e420d96b020ecf1a359588 Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Mon, 20 Sep 2021 13:22:18 -0700 Subject: [PATCH 025/229] modified description --- _clients/go.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_clients/go.md b/_clients/go.md index 6db260ac..46b359aa 100644 --- a/_clients/go.md +++ b/_clients/go.md @@ -6,7 +6,7 @@ nav_order: 80 # Go client -The OpenSearch Go client lets you programmatically interact with data in your OpenSearch cluster as part of your Go application. +The OpenSearch Go client lets you build a Go application that interacts with the data in your OpenSearch cluster. ## Setup From 6bf8a7d51aa1a745685d82fc7ab03d20f3b8c828 Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Mon, 20 Sep 2021 13:27:13 -0700 Subject: [PATCH 026/229] minor change --- _clients/go.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_clients/go.md b/_clients/go.md index 46b359aa..ac09a0c8 100644 --- a/_clients/go.md +++ b/_clients/go.md @@ -6,7 +6,7 @@ nav_order: 80 # Go client -The OpenSearch Go client lets you build a Go application that interacts with the data in your OpenSearch cluster. +The OpenSearch Go client lets you easily connect your Go application with the data in your OpenSearch cluster. ## Setup From 54a0b8e755b2f4873f99163949fd60fd0780f96d Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Mon, 20 Sep 2021 13:29:58 -0700 Subject: [PATCH 027/229] minor change --- _clients/go.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_clients/go.md b/_clients/go.md index ac09a0c8..75ee300e 100644 --- a/_clients/go.md +++ b/_clients/go.md @@ -6,7 +6,7 @@ nav_order: 80 # Go client -The OpenSearch Go client lets you easily connect your Go application with the data in your OpenSearch cluster. +The OpenSearch Go client lets you connect your Go application with the data in your OpenSearch cluster. ## Setup From 3c98e4b2970464d5c04e0bd2d4d261e07340e463 Mon Sep 17 00:00:00 2001 From: keithhc2 Date: Mon, 20 Sep 2021 16:06:09 -0700 Subject: [PATCH 028/229] Modified monitoring to have two kinds of monitors --- _monitoring-plugins/alerting/api.md | 268 ++++++++++++++++++++++- _monitoring-plugins/alerting/monitors.md | 81 ++++--- 2 files changed, 311 insertions(+), 38 deletions(-) diff --git a/_monitoring-plugins/alerting/api.md b/_monitoring-plugins/alerting/api.md index 0799878a..146f2d9a 100644 --- a/_monitoring-plugins/alerting/api.md +++ b/_monitoring-plugins/alerting/api.md @@ -19,10 +19,11 @@ Use the alerting API to programmatically manage monitors and alerts. --- -## Create monitor +## Create query-level monitor Introduced 1.0 {: .label .label-purple } + #### Request ```json @@ -30,6 +31,7 @@ POST _plugins/_alerting/monitors { "type": "monitor", "name": "test-monitor", + "monitor_type": "query_level_monitor", "enabled": true, "schedule": { "period": { @@ -166,7 +168,7 @@ If you use a custom webhook for your destination and need to embed JSON in the m }, "throttle_enabled": false, "subject_template": { - "source": "TheSubject", + "source": "Subject", "lang": "mustache" } }] @@ -186,6 +188,7 @@ The following example creates a monitor that runs at 12:10 PM Pacific Time on th { "type": "monitor", "name": "test-monitor", + "monitor_type": "query_level_monitor", "enabled": true, "schedule": { "cron" : { @@ -228,7 +231,7 @@ The following example creates a monitor that runs at 12:10 PM Pacific Time on th "name": "test-action", "destination_id": "ld7912sBlQ5JUWWFThoW", "message_template": { - "source": "This is my message body." + "source": "This is a message body." }, "throttle_enabled": true, "throttle": { @@ -236,7 +239,7 @@ The following example creates a monitor that runs at 12:10 PM Pacific Time on th "unit": "MINUTES" }, "subject_template": { - "source": "TheSubject" + "source": "Subject" } }] }] @@ -247,6 +250,263 @@ For a full list of timezone names, refer to [Wikipedia](https://en.wikipedia.org --- +## Create bucket-level monitor + +```json +POST _plugins/_alerting/monitors +{ + "type": "monitor", + "name": "test-bucket-level-monitor", + "monitor_type": "bucket_level_monitor", + "enabled": true, + "schedule": { + "period": { + "interval": 1, + "unit": "MINUTES" + } + }, + "inputs": [ + { + "search": { + "indices": [ + "movies" + ], + "query": { + "size": 0, + "query": { + "bool": { + "filter": [ + { + "range": { + "order_date": { + "from": "{{period_end}}||-1h", + "to": "{{period_end}}", + "include_lower": true, + "include_upper": true, + "format": "epoch_millis" + } + } + } + ] + } + }, + "aggregations": { + "composite_agg": { + "composite": { + "sources": [ + { + "user": { + "terms": { + "field": "user" + } + } + } + ] + }, + "aggregations": { + "avg_products_base_price": { + "avg": { + "field": "products.base_price" + } + } + } + } + } + } + } + } + ], + "triggers": [ + { + "bucket_level_trigger": { + "name": "test-trigger", + "severity": "1", + "condition": { + "buckets_path": { + "_count": "_count", + "avg_products_base_price": "avg_products_base_price" + }, + "parent_bucket_path": "composite_agg", + "script": { + "source": "params._count > 50 || params.avg_products_base_price < 35", + "lang": "painless" + } + }, + "actions": [ + { + "name": "test-action", + "destination_id": "E4o5hnsB6KjPKmHtpfCA", + "message_template": { + "source": """Monitor {{ctx.monitor.name}} just entered alert status. Please investigate the issue. - Trigger: {{ctx.trigger.name}} - Severity: {{ctx.trigger.severity}} - Period start: {{ctx.periodStart}} - Period end: {{ctx.periodEnd}} - Deduped Alerts: {{ctx.dedupedAlerts}} * {{id}} : {{bucket_keys}} {{ctx.dedupedAlerts}} - New Alerts: {{ctx.newAlerts}} * {{id}} : {{bucket_keys}} {{ctx.newAlerts}} - Completed Alerts: {{ctx.completedAlerts}} * {{id}} : {{bucket_keys}} {{ctx.completedAlerts}}""", + "lang": "mustache" + }, + "throttle_enabled": false, + "action_execution_policy": { + "throttle": { + "value": 10, + "unit": "MINUTES" + }, + "action_execution_scope": { + "per_alert": { + "actionable_alerts": [ + "DEDUPED", + "NEW" + ] + } + } + }, + "subject_template": { + "source": "Sample subject", + "lang": "mustache" + } + } + ] + } + } + ] +} +``` + +#### Sample response +```json +{ + "_id" : "Dfxr63sBwex6DxEhHV5N", + "_version" : 1, + "_seq_no" : 3, + "_primary_term" : 1, + "monitor" : { + "type" : "monitor", + "schema_version" : 4, + "name" : "test-bucket-level-monitor", + "monitor_type" : "bucket_level_monitor", + "user" : { + "name" : "", + "backend_roles" : [ ], + "roles" : [ ], + "custom_attribute_names" : [ ], + "user_requested_tenant" : null + }, + "enabled" : true, + "enabled_time" : 1631742270785, + "schedule" : { + "period" : { + "interval" : 1, + "unit" : "MINUTES" + } + }, + "inputs" : [ + { + "search" : { + "indices" : [ + "opensearch_dashboards_sample_data_flights" + ], + "query" : { + "size" : 0, + "query" : { + "bool" : { + "filter" : [ + { + "range" : { + "order_date" : { + "from" : "{{period_end}}||-1h", + "to" : "{{period_end}}", + "include_lower" : true, + "include_upper" : true, + "format" : "epoch_millis", + "boost" : 1.0 + } + } + } + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + }, + "aggregations" : { + "composite_agg" : { + "composite" : { + "size" : 10, + "sources" : [ + { + "user" : { + "terms" : { + "field" : "user", + "missing_bucket" : false, + "order" : "asc" + } + } + } + ] + }, + "aggregations" : { + "avg_products_base_price" : { + "avg" : { + "field" : "products.base_price" + } + } + } + } + } + } + } + } + ], + "triggers" : [ + { + "bucket_level_trigger" : { + "id" : "C_xr63sBwex6DxEhHV5B", + "name" : "test-trigger", + "severity" : "1", + "condition" : { + "buckets_path" : { + "_count" : "_count", + "avg_products_base_price" : "avg_products_base_price" + }, + "parent_bucket_path" : "composite_agg", + "script" : { + "source" : "params._count > 50 || params.avg_products_base_price < 35", + "lang" : "painless" + }, + "gap_policy" : "skip" + }, + "actions" : [ + { + "id" : "DPxr63sBwex6DxEhHV5B", + "name" : "test-action", + "destination_id" : "E4o5hnsB6KjPKmHtpfCA", + "message_template" : { + "source" : "Monitor {{ctx.monitor.name}} just entered alert status. Please investigate the issue. - Trigger: {{ctx.trigger.name}} - Severity: {{ctx.trigger.severity}} - Period start: {{ctx.periodStart}} - Period end: {{ctx.periodEnd}} - Deduped Alerts: {{ctx.dedupedAlerts}} * {{id}} : {{bucket_keys}} {{ctx.dedupedAlerts}} - New Alerts: {{ctx.newAlerts}} * {{id}} : {{bucket_keys}} {{ctx.newAlerts}} - Completed Alerts: {{ctx.completedAlerts}} * {{id}} : {{bucket_keys}} {{ctx.completedAlerts}}", + "lang" : "mustache" + }, + "throttle_enabled" : false, + "subject_template" : { + "source" : "The Subject", + "lang" : "mustache" + }, + "throttle" : { + "value" : 10, + "unit" : "MINUTES" + }, + "action_execution_policy" : { + "action_execution_scope" : { + "per_alert" : { + "actionable_alerts" : [ + "DEDUPED", + "NEW" + ] + } + } + } + } + ] + } + } + ], + "last_update_time" : 1631742270785 + } +} +``` + ## Update monitor Introduced 1.0 {: .label .label-purple } diff --git a/_monitoring-plugins/alerting/monitors.md b/_monitoring-plugins/alerting/monitors.md index 85138490..085eafc7 100644 --- a/_monitoring-plugins/alerting/monitors.md +++ b/_monitoring-plugins/alerting/monitors.md @@ -19,11 +19,11 @@ has_children: false Term | Definition :--- | :--- -Monitor | A job that runs on a defined schedule and queries OpenSearch. The results of these queries are then used as input for one or more *triggers*. +Monitor | A job that runs on a defined schedule and queries OpenSearch indices. The results of these queries are then used as input for one or more *triggers*. Trigger | Conditions that, if met, generate *alerts*. Alert | An event associated with a trigger. When an alert is created, the trigger performs *actions*, which can include sending a notification. Action | The information that you want the monitor to send out after being triggered. Actions have a *destination*, a message subject, and a message body. -Destination | A reusable location for an action, such as Amazon Chime, Slack, or a webhook URL. +Destination | A reusable location for an action. Supported locations are Amazon Chime, Email, Slack, or custom webhook. --- @@ -34,9 +34,9 @@ Destination | A reusable location for an action, such as Amazon Chime, Slack, or 1. Specify a name for the destination so that you can identify it later. 1. For **Type**, choose Slack, Amazon Chime, custom webhook, or [email](#email-as-a-destination). -For Email type, refer to [Email as a destination](#email-as-a-destination) section below. For all other types, specify the webhook URL. For more information about webhooks, see the documentation for [Slack](https://api.slack.com/incoming-webhooks) and [Amazon Chime](https://docs.aws.amazon.com/chime/latest/ug/webhooks.html). +For Email, refer to the [Email as a destination](#email-as-a-destination) section below. For all other types, specify the webhook URL. See the documentation for [Slack](https://api.slack.com/incoming-webhooks) and [Amazon Chime](https://docs.aws.amazon.com/chime/latest/ug/webhooks.html) to learn more about webhooks. -For custom webhooks, you must specify more information: parameters and headers. For example, if your endpoint requires basic authentication, you might need to add a header with a key of `Authorization` and a value of `Basic `. You might also need to change `Content-Type` to whatever your webhook requires. Popular values are `application/json`, `application/xml`, and `text/plain`. +If you're using custom webhooks, you must specify more information: parameters and headers. For example, if your endpoint requires basic authentication, you might need to add a header with a key of `Authorization` and a value of `Basic `. You might also need to change `Content-Type` to whatever your webhook requires. Popular values are `application/json`, `application/xml`, and `text/plain`. This information is stored in plain text in the OpenSearch cluster. We will improve this design in the future, but for now, the encoded credentials (which are neither encrypted nor hashed) might be visible to other OpenSearch users. @@ -55,7 +55,7 @@ To configure a sender email, do the following: 1. After you choose **Email** as the destination type, choose **Manage senders**. 1. Choose **Add sender**, **New sender** and enter a unique name. 1. Enter the email address, SMTP host (e.g. `smtp.gmail.com` for a Gmail account), and the port. -1. Choose an encryption method, or use the default value of **None**. However, most email providers require SSL or TLS, which requires a username and password in OpenSearch keystore. Refer to [Authenticate sender account](#authenticate-sender-account) to learn more. +1. Choose an encryption method, or use the default value of **None**. However, most email providers require SSL or TLS, which require a username and password in OpenSearch keystore. Refer to [Authenticate sender account](#authenticate-sender-account) to learn more. 1. Choose **Save** to save the configuration and create the sender. You can create a sender even before you add your credentials to the OpenSearch keystore. However, you must [authenticate each sender account](#authenticate-sender-account) before you use the destination to send your alert. You can reuse senders across many different destinations, but each destination only supports one sender. @@ -101,20 +101,9 @@ POST _nodes/reload_secure_settings 1. Choose **Alerting**, **Monitors**, **Create monitor**. 1. Specify a name for the monitor. +1. Choose either **Per query monitor** or **Per bucket monitor**. -The anomaly detection option is for pairing with the anomaly detection plugin. See [Anomaly Detection]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/). -For anomaly detector, choose an appropriate schedule for the monitor based on the detector interval. Otherwise, the alerting monitor might miss reading the results. - -For example, assume you set the monitor interval and the detector interval as 5 minutes, and you start the detector at 12:00. If an anomaly is detected at 12:05, it might be available at 12:06 because of the delay between writing the anomaly and it being available for queries. The monitor reads the anomaly results between 12:00 and 12:05, so it does not get the anomaly results available at 12:06. - -To avoid this issue, make sure the alerting monitor is at least twice the detector interval. -When you create a monitor using OpenSearch Dashboards, the anomaly detector plugin generates a default monitor schedule that's twice the detector interval. - -Whenever you update a detector’s interval, make sure to update the associated monitor interval as well, as the anomaly detection plugin does not do this automatically. - -1. Choose one or more indices. You can also use `*` as a wildcard to specify an index pattern. - - If you use the security plugin, you can only choose indices that you have permission to access. For details, see [Alerting security]({{site.url}}{{site.baseurl}}/security-plugin/). +Whereas per query monitors run your specifed query and then check whether the query's results triggers any alerts, per bucket monitors let you select fields to create buckets and categorize your results into those buckets. Doing so gives you finer control over which results should trigger alerts, and trigger conditions get evaluated per bucket. 1. Define the monitor in one of three ways: visually, using a query, or using an anomaly detector. @@ -170,27 +159,45 @@ Whenever you update a detector’s interval, make sure to update the associated "Start" and "end" refer to the interval at which the monitor runs. See [Available variables](#available-variables). -1. To define a monitor visually, choose **Define using visual graph**. Then choose an aggregation (for example, `count()` or `average()`), a set of documents, and a timeframe. Visual definition works well for most monitors. - - To use a query, choose **Define using extraction query**, add your query (using [the OpenSearch query DSL]({{site.url}}{{site.baseurl}}/opensearch/query-dsl/full-text/)), and test it using the **Run** button. - - The monitor makes this query to OpenSearch as often as the schedule dictates; check the **Query Performance** section and make sure you're comfortable with the performance implications. - - To use an anomaly detector, choose **Define using Anomaly detector** and select your **Detector**. 1. Choose a frequency and timezone for your monitor. Note that you can only pick a timezone if you choose Daily, Weekly, Monthly, or [custom cron expression]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/cron/) for frequency. -1. Choose **Create**. +1. Choose one or more indices. You can also use `*` as a wildcard to specify an index pattern. + + If you use the security plugin, you can only choose indices that you have permission to access. For details, see [Alerting security]({{site.url}}{{site.baseurl}}/security-plugin/). + + To define a monitor visually, choose **Visual editor**. Then choose an aggregation (for example, `count()` or `average()`), a set of documents, a timeframe, a data filter if you want to monitor a subset of your source index, and a group-by field if you want to categorize your query results into separate buckets, and trigger conditions get evaluated per bucket. At least one group-by field is required if you are creating a per bucket monitor. Visual definition works well for most monitors. + + To use a query, choose **Extraction query editor**, add your query (using [the OpenSearch query DSL]({{site.url}}{{site.baseurl}}/opensearch/query-dsl/full-text/)), and test it using the **Run** button. + + The monitor makes this query to OpenSearch as often as the schedule dictates; check the **Query Performance** section and make sure you're comfortable with the performance implications. + + To use an anomaly detector, choose **Anomaly detector** and select your **Detector**. + + The anomaly detection option is for pairing with the anomaly detection plugin. See [Anomaly Detection]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/). + For anomaly detector, choose an appropriate schedule for the monitor based on the detector interval. Otherwise, the alerting monitor might miss reading the results. + + For example, assume you set the monitor interval and the detector interval as 5 minutes, and you start the detector at 12:00. If an anomaly is detected at 12:05, it might be available at 12:06 because of the delay between writing the anomaly and it being available for queries. The monitor reads the anomaly results between 12:00 and 12:05, so it does not get the anomaly results available at 12:06. + + To avoid this issue, make sure the alerting monitor is at least twice the detector interval. + When you create a monitor using OpenSearch Dashboards, the anomaly detector plugin generates a default monitor schedule that's twice the detector interval. + + Whenever you update a detector’s interval, make sure to update the associated monitor interval as well, as the anomaly detection plugin does not do this automatically. + + **Note**: Anomaly detection is available only if you are defining a per query monitor. + {: .note} + +1. Add a trigger to your monitor. --- ## Create triggers -The next step in creating a monitor is to create a trigger. These steps differ depending on whether you chose **Define using visual graph** or **Define using extraction query** or **Define using Anomaly detector** when you created the monitor. +Steps to create a trigger differ depending on whether you chose **Visual editor**, **Extraction query editor**, or **Anomaly detector** when you created the monitor. -Either way, you begin by specifying a name and severity level for the trigger. Severity levels help you manage alerts. A trigger with a high severity level (e.g. 1) might page a specific individual, whereas a trigger with a low severity level might message a chat room. +You begin by specifying a name and severity level for the trigger. Severity levels help you manage alerts. A trigger with a high severity level (e.g. 1) might page a specific individual, whereas a trigger with a low severity level might message a chat room. -### Visual graph +### Visual editor For **Trigger condition**, specify a threshold for the aggregation and timeframe you chose earlier, such as "is below 1,000" or "is exactly 10." @@ -264,11 +271,11 @@ Below are some variables you can include in your message using Mustache template Variable | Data Type | Description :--- | :--- | :--- -`ctx.monitor` | JSON | Includes `ctx.monitor.name`, `ctx.monitor.type`, `ctx.monitor.enabled`, `ctx.monitor.enabled_time`, `ctx.monitor.schedule`, `ctx.monitor.inputs`, `triggers` and `ctx.monitor.last_update_time`. -`ctx.monitor.user` | JSON | Includes information about the user who created the monitor. Includes `ctx.monitor.user.backend_roles` and `ctx.monitor.user.roles`, which are arrays that contain the backend roles and roles assigned to the user. See [alerting security]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/security/) for more information. +`ctx.monitor` | Object | Includes `ctx.monitor.name`, `ctx.monitor.type`, `ctx.monitor.enabled`, `ctx.monitor.enabled_time`, `ctx.monitor.schedule`, `ctx.monitor.inputs`, `triggers` and `ctx.monitor.last_update_time`. +`ctx.monitor.user` | Object | Includes information about the user who created the monitor. Includes `ctx.monitor.user.backend_roles` and `ctx.monitor.user.roles`, which are arrays that contain the backend roles and roles assigned to the user. See [alerting security]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/security/) for more information. `ctx.monitor.enabled` | Boolean | Whether the monitor is enabled. `ctx.monitor.enabled_time` | Milliseconds | Unix epoch time of when the monitor was last enabled. -`ctx.monitor.schedule` | JSON | Contains a schedule of how often or when the monitor should run. +`ctx.monitor.schedule` | Object | Contains a schedule of how often or when the monitor should run. `ctx.monitor.schedule.period.interval` | Integer | The interval at which the monitor runs. `ctx.monitor.schedule.period.unit` | String | The interval's unit of time. `ctx.monitor.inputs` | Array | An array that contains the indices and definition used to create the monitor. @@ -282,7 +289,7 @@ Variable | Data Type | Description `ctx.trigger.id` | String | The trigger's ID. `ctx.trigger.name` | String | The trigger's name. `ctx.trigger.severity` | String | The trigger's severity. -`ctx.trigger.condition`| JSON | Contains the Painless script used when creating the monitor. +`ctx.trigger.condition`| Object | Contains the Painless script used when creating the monitor. `ctx.trigger.condition.script.source` | String | The language used to define the script. Must be painless. `ctx.trigger.condition.script.lang` | String | The script used to define the trigger. `ctx.trigger.actions`| Array | An array with one element that contains information about the action the monitor needs to trigger. @@ -309,7 +316,13 @@ Variable | Data Type | Description `ctx.periodStart` | String | Unix timestamp for the beginning of the period during which the alert triggered. For example, if a monitor runs every ten minutes, a period might begin at 10:40 and end at 10:50. `ctx.periodEnd` | String | The end of the period during which the alert triggered. `ctx.error` | String | The error message if the trigger was unable to retrieve results or unable to evaluate the trigger, typically due to a compile error or null pointer exception. Null otherwise. -`ctx.alert` | JSON | The current, active alert (if it exists). Includes `ctx.alert.id`, `ctx.alert.version`, and `ctx.alert.isAcknowledged`. Null if no alert is active. +`ctx.alert` | Object | The current, active alert (if it exists). Includes `ctx.alert.id`, `ctx.alert.version`, and `ctx.alert.isAcknowledged`. Null if no alert is active. +`ctx.dedupedAlerts` | Object | Alerts that have already been triggered. OpenSearch keeps the existing alert to prevent the plugin from creating endless amounts of the same alerts. Only available with bucket-level monitors. +`ctx.newAlerts` | Object | Newly created alerts. Only available with bucket-level monitors. +`ctx.completedAlerts` | Object | Alerts that are no longer ongoing. Only available with bucket-level monitors. +`bucket_keys` | String | Comma-separated list of the monitor's bucket key values. Available only for `ctx.dedupedAlerts`, `ctx.newAlerts`, and `ctx.completedAlerts`. Accessed through `ctx.dedupedAlerts[0].bucket_keys`. +`parent_bucket_path` | String | The parent bucket path of the bucket that triggered the alert. Accessed through `ctx.dedupedAlerts[0].parent_bucket_path`. + --- From 65f333d038325af731ff13dcae6e45f400d5398b Mon Sep 17 00:00:00 2001 From: aetter Date: Tue, 21 Sep 2021 13:40:38 -0700 Subject: [PATCH 029/229] Update python.md --- _clients/python.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/_clients/python.md b/_clients/python.md index bcb1ff93..9ce68bca 100644 --- a/_clients/python.md +++ b/_clients/python.md @@ -8,19 +8,23 @@ nav_order: 70 The OpenSearch Python client provides a more natural syntax for interacting with your cluster. Rather than sending HTTP requests with raw JSON bodies to a given URL, you can create an OpenSearch client for your cluster and call the client's built-in functions. +{% comment %} +`opensearch-py` is the lower-level of the two Python clients. If you want a general client for assorted operations, it's a great choice. If you want a higher-level client strictly for indexing and search operations, consider [opensearch-dsl-py]({{site.url}}{{site.baseurl}}/clients/python-dsl/). +{% endcomment %} + ## Setup To add the client to your project, install it using [pip](https://pip.pypa.io/): ```bash -pip install opensearch +pip install opensearch-py ``` Then import it like any other module: ```python -from opensearch import OpenSearch +from opensearchpy import OpenSearch ``` If you prefer to add the client manually or just want to examine the source code, see [opensearch-py on GitHub](https://github.com/opensearch-project/opensearch-py). @@ -29,7 +33,7 @@ If you prefer to add the client manually or just want to examine the source code ## Sample code ```python -from opensearch import OpenSearch +from opensearchpy import OpenSearch host = 'localhost' port = 9200 @@ -55,7 +59,7 @@ client = OpenSearch( ) # Create an index with non-default settings. -index_name = 'python-test-index3' +index_name = 'python-test-index' index_body = { 'settings': { 'index': { From 66ca05e7145c266b792938b461c686a964156566 Mon Sep 17 00:00:00 2001 From: aetter Date: Tue, 21 Sep 2021 13:56:25 -0700 Subject: [PATCH 030/229] Update links --- _clients/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_clients/index.md b/_clients/index.md index 3880a3bb..d920e16a 100644 --- a/_clients/index.md +++ b/_clients/index.md @@ -15,8 +15,8 @@ For example, a 1.0.0 client works with an OpenSearch 1.1.0 cluster, but might no * [OpenSearch Java client]({{site.url}}{{site.baseurl}}/clients/java/) * [OpenSearch Python client]({{site.url}}{{site.baseurl}}/clients/python/) -* [OpenSearch JavaScript (Node.js) client]({{site.url}}{{site.baseurl}}/clients/nodejs/) -* [OpenSearch Go client]({{site.url}}{{site.baseurl}}/clients/golang/) +* [OpenSearch JavaScript (Node.js) client]({{site.url}}{{site.baseurl}}/clients/javascript/) +* [OpenSearch Go client]({{site.url}}{{site.baseurl}}/clients/go/) ## Legacy clients From 2d228a86ca7b46fdba8bccfe39aada53728ef62a Mon Sep 17 00:00:00 2001 From: Liz Snyder Date: Wed, 22 Sep 2021 12:26:41 -0700 Subject: [PATCH 031/229] Update client name --- _clients/javascript.md | 163 ++++++++++++++++++++--------------------- 1 file changed, 80 insertions(+), 83 deletions(-) diff --git a/_clients/javascript.md b/_clients/javascript.md index 55134c00..5d429a08 100644 --- a/_clients/javascript.md +++ b/_clients/javascript.md @@ -10,19 +10,18 @@ The OpenSearch Javascript client provides a safer and easier way to interact wit The client contains a library of APIs that let you perform different operations on your cluster and return a standard response body. The example here demonstrates some basic operations like creating an index, adding documents, and searching your data. - ## Setup To add the client to your project, install it from npm: ```bash -npm install @opensearch/opensearch +npm install @opensearch-project/opensearch ``` To install a specific major version of the client, run the following command: ```bash -npm install @opensearch/opensearch@ +npm install @opensearch-project/opensearch@ ``` If you prefer to add the client manually or just want to examine the source code, see [opensearch-js](https://github.com/opensearch-project/opensearch-js) on GitHub. @@ -30,115 +29,113 @@ If you prefer to add the client manually or just want to examine the source code Then require the client: ```javascript -const { Client } = require('@opensearch/opensearch') +const { Client } = require("@opensearch-project/opensearch"); ``` - ## Sample code ```javascript -'use strict'; +"use strict"; -var host = 'localhost'; -var protocol = 'https'; +var host = "localhost"; +var protocol = "https"; var port = 9200; -var auth = 'admin:admin'; // For testing only. Don't store credentials in code. -var ca_certs_path = '/full/path/to/root-ca.pem'; +var auth = "admin:admin"; // For testing only. Don't store credentials in code. +var ca_certs_path = "/full/path/to/root-ca.pem"; // Optional client certificates if you don't want to use HTTP basic authentication. // var client_cert_path = '/full/path/to/client.pem' // var client_key_path = '/full/path/to/client-key.pem' // Create a client with SSL/TLS enabled. -var { Client } = require('@opensearch/opensearch'); -var fs = require('fs'); +var { Client } = require("@opensearch/opensearch"); +var fs = require("fs"); var client = new Client({ - node: protocol + '://' + auth + '@' + host + ':' + port, - ssl: { - ca: fs.readFileSync(ca_certs_path), - // You can turn off certificate verification (rejectUnauthorized: false) if you're using self-signed certificates with a hostname mismatch. - // cert: fs.readFileSync(client_cert_path), - // key: fs.readFileSync(client_key_path) - } -}) + node: protocol + "://" + auth + "@" + host + ":" + port, + ssl: { + ca: fs.readFileSync(ca_certs_path), + // You can turn off certificate verification (rejectUnauthorized: false) if you're using self-signed certificates with a hostname mismatch. + // cert: fs.readFileSync(client_cert_path), + // key: fs.readFileSync(client_key_path) + }, +}); async function search() { + // Create an index with non-default settings. + var index_name = "books"; + var settings = { + settings: { + index: { + number_of_shards: 4, + number_of_replicas: 3, + }, + }, + }; - // Create an index with non-default settings. - var index_name = 'books' - var settings = { - 'settings': { - 'index': { - 'number_of_shards': 4, - 'number_of_replicas': 3 - } - } - } + var response = await client.indices.create({ + index: index_name, + body: settings, + }); - var response = await client.indices.create({ - index: index_name, - body: settings - }) + console.log("Creating index:"); + console.log(response.body); - console.log('Creating index:') - console.log(response.body) + // Add a document to the index. + var document = { + title: "The Outsider", + author: "Stephen King", + year: "2018", + genre: "Crime fiction", + }; - // Add a document to the index. - var document = { - 'title': 'The Outsider', - 'author': 'Stephen King', - 'year': '2018', - 'genre': 'Crime fiction' - } + var id = "1"; - var id = '1' + var response = await client.index({ + id: id, + index: index_name, + body: document, + refresh: true, + }); - var response = await client.index({ - id: id, - index: index_name, - body: document, - refresh: true - }) + console.log("Adding document:"); + console.log(response.body); - console.log('Adding document:') - console.log(response.body) + // Search for the document. + var query = { + query: { + match: { + title: { + query: "The Outsider", + }, + }, + }, + }; - // Search for the document. - var query = { - 'query': { - 'match': { - 'title': { - 'query': 'The Outsider' - } - } - } - } + var response = await client.search({ + index: index_name, + body: query, + }); - var response = await client.search({ - index: index_name, - body: query - }) + console.log("Search results:"); + console.log(response.body.hits); - console.log('Search results:') - console.log(response.body.hits) + // Delete the document. + var response = await client.delete({ + index: index_name, + id: id, + }); - // Delete the document. - var response = await client.delete({ - index: index_name, - id: id - }) + console.log("Deleting document:"); + console.log(response.body); - console.log('Deleting document:') - console.log(response.body) + // Delete the index. + var response = await client.indices.delete({ + index: index_name, + }); - // Delete the index. - var response = await client.indices.delete({ - index: index_name - }) - - console.log('Deleting index:') - console.log(response.body) + console.log("Deleting index:"); + console.log(response.body); } -search().catch(console.log) +search().catch(console.log); ``` From 293743e15b249fb45d8a985b481058bd20be84cf Mon Sep 17 00:00:00 2001 From: Liz Snyder Date: Wed, 22 Sep 2021 12:27:39 -0700 Subject: [PATCH 032/229] Missed one --- _clients/javascript.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_clients/javascript.md b/_clients/javascript.md index 5d429a08..cb308a29 100644 --- a/_clients/javascript.md +++ b/_clients/javascript.md @@ -48,7 +48,7 @@ var ca_certs_path = "/full/path/to/root-ca.pem"; // var client_key_path = '/full/path/to/client-key.pem' // Create a client with SSL/TLS enabled. -var { Client } = require("@opensearch/opensearch"); +var { Client } = require("@opensearch-project/opensearch"); var fs = require("fs"); var client = new Client({ node: protocol + "://" + auth + "@" + host + ":" + port, From a3d3b183c586d55879482476b0141d3c85bbdb8b Mon Sep 17 00:00:00 2001 From: Naveen Tatikonda Date: Mon, 27 Sep 2021 11:34:39 -0500 Subject: [PATCH 033/229] Adding sample configuration for auth_type and aws_iam Signed-off-by: Naveen Tatikonda --- _clients/logstash/ship-to-opensearch.md | 69 +++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/_clients/logstash/ship-to-opensearch.md b/_clients/logstash/ship-to-opensearch.md index e9ef7260..23cbfeea 100644 --- a/_clients/logstash/ship-to-opensearch.md +++ b/_clients/logstash/ship-to-opensearch.md @@ -75,3 +75,72 @@ output { health | status | index | uuid | pri | rep | docs.count | docs.deleted | store.size | pri.store.size green | open | logstash-logs-2021.07.01 | iuh648LYSnmQrkGf70pplA | 1 | 1 | 1 | 0 | 10.3kb | 5.1kb ``` + +## Adding different Authentication mechanisms in the Output plugin + +## auth_type to support different authentication mechanisms + +In addition to the existing authentication mechanisms, if we want to add new authentication then we will be adding them in the configuration by using auth_type + +Example Configuration for basic authentication: + +```yml +output { + opensearch { + hosts => ["https://hostname:port"] + auth_type => { + type => 'basic' + user => 'admin' + password => 'admin' + } + index => "logstash-logs-%{+YYYY.MM.dd}" + } +} +``` +### Parameters inside auth_type + +- type (string) - We should specify the type of authentication +- We should add credentials required for that authentication like 'user' and 'password' for 'basic' authentication +- We should also add other parameters required for that authentication mechanism like we added 'region' for 'aws_iam' authentication + +## Configuration for AWS IAM Authentication + +To run the Logstash Output Opensearch plugin using aws_iam authentication, simply add a configuration following the below documentation. + +Example Configuration: + +```yml +output { + opensearch { + hosts => ["https://hostname:port"] + auth_type => { + type => 'aws_iam' + aws_access_key_id => 'ACCESS_KEY' + aws_secret_access_key => 'SECRET_KEY' + region => 'us-west-2' + } + index => "logstash-logs-%{+YYYY.MM.dd}" + } +} +``` + +### Required Parameters + +- hosts (array of string) - AmazonOpensearchService domain endpoint : port number +- auth_type (Json object) - Which holds other parameters required for authentication + - type (string) - "aws_iam" + - aws_access_key_id (string) - AWS access key + - aws_secret_access_key (string) - AWS secret access key + - region (string, :default => "us-east-1") - region in which the domain is located + - if we want to pass other optional parameters like profile, session_token,etc. They needs to be added in auth_type +- port (string) - AmazonOpensearchService listens on port 443 for HTTPS +- protocol (string) - The protocol used to connect to AmazonOpensearchService is 'https' + +### Optional Parameters +- The credential resolution logic can be described as follows: + - User passed aws_access_key_id and aws_secret_access_key in configuration + - Environment variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY (RECOMMENDED since they are recognized by all the AWS SDKs and CLI except for .NET), or AWS_ACCESS_KEY and AWS_SECRET_KEY (only recognized by Java SDK) + - Credential profiles file at the default location (~/.aws/credentials) shared by all AWS SDKs and the AWS CLI + - Instance profile credentials delivered through the Amazon EC2 metadata service +- template (path) - You can set the path to your own template here, if you so desire. If not set, the included template will be used. +- template_name (string, default => "logstash") - defines how the template is named inside Opensearch From a0590a400c08fce0a8e485d7c888a6918edab944 Mon Sep 17 00:00:00 2001 From: Keith Chan <12404772+keithhc2@users.noreply.github.com> Date: Mon, 27 Sep 2021 10:09:34 -0700 Subject: [PATCH 034/229] Added maven repo --- _clients/java.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/_clients/java.md b/_clients/java.md index 3823d0c4..06a63692 100644 --- a/_clients/java.md +++ b/_clients/java.md @@ -14,7 +14,9 @@ To start using the OpenSearch Java client, ensure that you have the following de ``` - + org.opensearch.client + opensearch-rest-client + 1.0.0 ``` From b4642195e72df21fdbcb01b88cf2adb02ed63882 Mon Sep 17 00:00:00 2001 From: keithhc2 Date: Mon, 27 Sep 2021 16:09:53 -0700 Subject: [PATCH 035/229] Addressed comments --- _monitoring-plugins/alerting/api.md | 3 +++ _monitoring-plugins/alerting/monitors.md | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/_monitoring-plugins/alerting/api.md b/_monitoring-plugins/alerting/api.md index 146f2d9a..265954ec 100644 --- a/_monitoring-plugins/alerting/api.md +++ b/_monitoring-plugins/alerting/api.md @@ -23,6 +23,7 @@ Use the alerting API to programmatically manage monitors and alerts. Introduced 1.0 {: .label .label-purple } +Query-level monitors run the query and check whether the results should trigger any alerts. For more information about query-level monitors versus bucket-level monitors, see [Create monitors]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/monitors/#create-monitors). #### Request @@ -252,6 +253,8 @@ For a full list of timezone names, refer to [Wikipedia](https://en.wikipedia.org ## Create bucket-level monitor +Bucket-level monitors categorize results into buckets separated by fields. For more information about bucket-level monitors versus query-level monitors, see [Create monitors]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/monitors/#create-monitors). + ```json POST _plugins/_alerting/monitors { diff --git a/_monitoring-plugins/alerting/monitors.md b/_monitoring-plugins/alerting/monitors.md index 085eafc7..7b6c999a 100644 --- a/_monitoring-plugins/alerting/monitors.md +++ b/_monitoring-plugins/alerting/monitors.md @@ -82,7 +82,7 @@ If your email provider requires SSL or TLS, you must authenticate each sender ac ./bin/opensearch-keystore add plugins.alerting.destination.email..password ``` -**Note**: Keystore settings are node-specific. You must run these commands on each node. +Note: Keystore settings are node-specific. You must run these commands on each node. {: .note} To change or update your credentials (after you've added them to the keystore on every node), call the reload API to automatically update those credentials without restarting OpenSearch: @@ -103,7 +103,7 @@ POST _nodes/reload_secure_settings 1. Specify a name for the monitor. 1. Choose either **Per query monitor** or **Per bucket monitor**. -Whereas per query monitors run your specifed query and then check whether the query's results triggers any alerts, per bucket monitors let you select fields to create buckets and categorize your results into those buckets. Doing so gives you finer control over which results should trigger alerts, and trigger conditions get evaluated per bucket. +Whereas per-query monitors run your specified query and then check whether the query's results triggers any alerts, per-bucket monitors let you select fields to create buckets and categorize your results into those buckets. Doing so gives you finer control over which results should trigger alerts, as the alerting plugin uses each bucket's results to see if they should trigger any alerts. 1. Define the monitor in one of three ways: visually, using a query, or using an anomaly detector. From 7ed1c64a3569abeb6be92b58344039bcf0a5c334 Mon Sep 17 00:00:00 2001 From: keithhc2 Date: Tue, 28 Sep 2021 16:06:54 -0700 Subject: [PATCH 036/229] Added and adjusted some more language to address comments --- _monitoring-plugins/alerting/api.md | 14 +++++++------- _monitoring-plugins/alerting/monitors.md | 16 +++++++--------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/_monitoring-plugins/alerting/api.md b/_monitoring-plugins/alerting/api.md index 265954ec..07bdc682 100644 --- a/_monitoring-plugins/alerting/api.md +++ b/_monitoring-plugins/alerting/api.md @@ -23,7 +23,7 @@ Use the alerting API to programmatically manage monitors and alerts. Introduced 1.0 {: .label .label-purple } -Query-level monitors run the query and check whether the results should trigger any alerts. For more information about query-level monitors versus bucket-level monitors, see [Create monitors]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/monitors/#create-monitors). +Query-level monitors run the query and check whether the results should trigger any alerts. As such, query-level monitors can only trigger one alert at a time. For more information about query-level monitors versus bucket-level monitors, see [Create monitors]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/monitors/#create-monitors). #### Request @@ -253,7 +253,7 @@ For a full list of timezone names, refer to [Wikipedia](https://en.wikipedia.org ## Create bucket-level monitor -Bucket-level monitors categorize results into buckets separated by fields. For more information about bucket-level monitors versus query-level monitors, see [Create monitors]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/monitors/#create-monitors). +Bucket-level monitors categorize results into buckets separated by fields. The monitor then runs your script with each bucket's results and evaluates whether to trigger an alert. For more information about bucket-level monitors versus query-level monitors, see [Create monitors]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/monitors/#create-monitors). ```json POST _plugins/_alerting/monitors @@ -344,11 +344,11 @@ POST _plugins/_alerting/monitors "lang": "mustache" }, "throttle_enabled": false, + "throttle": { + "value": 10, + "unit": "MINUTES" + }, "action_execution_policy": { - "throttle": { - "value": 10, - "unit": "MINUTES" - }, "action_execution_scope": { "per_alert": { "actionable_alerts": [ @@ -359,7 +359,7 @@ POST _plugins/_alerting/monitors } }, "subject_template": { - "source": "Sample subject", + "source": "The Subject", "lang": "mustache" } } diff --git a/_monitoring-plugins/alerting/monitors.md b/_monitoring-plugins/alerting/monitors.md index 7b6c999a..c7f4284d 100644 --- a/_monitoring-plugins/alerting/monitors.md +++ b/_monitoring-plugins/alerting/monitors.md @@ -103,7 +103,7 @@ POST _nodes/reload_secure_settings 1. Specify a name for the monitor. 1. Choose either **Per query monitor** or **Per bucket monitor**. -Whereas per-query monitors run your specified query and then check whether the query's results triggers any alerts, per-bucket monitors let you select fields to create buckets and categorize your results into those buckets. Doing so gives you finer control over which results should trigger alerts, as the alerting plugin uses each bucket's results to see if they should trigger any alerts. +Whereas per-query monitors run your specified query and then check whether the query's results triggers any alerts, per-bucket monitors let you select fields to create buckets and categorize your results into those buckets. The alerting plugin runs each bucket's unique results against a script you define later, so you have finer control over which results should trigger alerts. Each of those buckets can trigger an alert, but per-query monitors can only trigger one alert at a time. 1. Define the monitor in one of three ways: visually, using a query, or using an anomaly detector. @@ -156,17 +156,12 @@ Whereas per-query monitors run your specified query and then check whether the q } ``` - "Start" and "end" refer to the interval at which the monitor runs. See [Available variables](#available-variables). + "Start" and "end" refer to the interval at which the monitor runs. See [Available variables](#available-variables). - -1. Choose a frequency and timezone for your monitor. Note that you can only pick a timezone if you choose Daily, Weekly, Monthly, or [custom cron expression]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/cron/) for frequency. - -1. Choose one or more indices. You can also use `*` as a wildcard to specify an index pattern. + To define a monitor visually, choose **Visual editor**. Then choose a source index, a timeframe, an aggregation (for example, `count()` or `average()`), a data filter if you want to monitor a subset of your source index, and a group-by field if you want to include an aggregation field in your query. Visual definition works well for most monitors. If you use the security plugin, you can only choose indices that you have permission to access. For details, see [Alerting security]({{site.url}}{{site.baseurl}}/security-plugin/). - To define a monitor visually, choose **Visual editor**. Then choose an aggregation (for example, `count()` or `average()`), a set of documents, a timeframe, a data filter if you want to monitor a subset of your source index, and a group-by field if you want to categorize your query results into separate buckets, and trigger conditions get evaluated per bucket. At least one group-by field is required if you are creating a per bucket monitor. Visual definition works well for most monitors. - To use a query, choose **Extraction query editor**, add your query (using [the OpenSearch query DSL]({{site.url}}{{site.baseurl}}/opensearch/query-dsl/full-text/)), and test it using the **Run** button. The monitor makes this query to OpenSearch as often as the schedule dictates; check the **Query Performance** section and make sure you're comfortable with the performance implications. @@ -186,6 +181,8 @@ Whereas per-query monitors run your specified query and then check whether the q **Note**: Anomaly detection is available only if you are defining a per query monitor. {: .note} +1. Choose a frequency and timezone for your monitor. Note that you can only pick a timezone if you choose Daily, Weekly, Monthly, or [custom cron expression]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/cron/) for frequency. + 1. Add a trigger to your monitor. --- @@ -196,6 +193,7 @@ Steps to create a trigger differ depending on whether you chose **Visual editor* You begin by specifying a name and severity level for the trigger. Severity levels help you manage alerts. A trigger with a high severity level (e.g. 1) might page a specific individual, whereas a trigger with a low severity level might message a chat room. +Remember that per-query monitors run your trigger's script just once against the query's results, but per-bucket monitors execute your trigger's script on each bucket, so you should create a trigger that best fits the monitor you chose. If you want to execute multiple scripts, you must create multiple triggers. ### Visual editor @@ -316,7 +314,7 @@ Variable | Data Type | Description `ctx.periodStart` | String | Unix timestamp for the beginning of the period during which the alert triggered. For example, if a monitor runs every ten minutes, a period might begin at 10:40 and end at 10:50. `ctx.periodEnd` | String | The end of the period during which the alert triggered. `ctx.error` | String | The error message if the trigger was unable to retrieve results or unable to evaluate the trigger, typically due to a compile error or null pointer exception. Null otherwise. -`ctx.alert` | Object | The current, active alert (if it exists). Includes `ctx.alert.id`, `ctx.alert.version`, and `ctx.alert.isAcknowledged`. Null if no alert is active. +`ctx.alert` | Object | The current, active alert (if it exists). Includes `ctx.alert.id`, `ctx.alert.version`, and `ctx.alert.isAcknowledged`. Null if no alert is active. Only available with per-query monitors. `ctx.dedupedAlerts` | Object | Alerts that have already been triggered. OpenSearch keeps the existing alert to prevent the plugin from creating endless amounts of the same alerts. Only available with bucket-level monitors. `ctx.newAlerts` | Object | Newly created alerts. Only available with bucket-level monitors. `ctx.completedAlerts` | Object | Alerts that are no longer ongoing. Only available with bucket-level monitors. From a186b4630211f1e56b91366ad23e96cf8c3e80d6 Mon Sep 17 00:00:00 2001 From: keithhc2 Date: Thu, 30 Sep 2021 12:00:37 -0700 Subject: [PATCH 037/229] Added trigger instructions for bucket-level monitors --- _monitoring-plugins/alerting/monitors.md | 36 ++++++++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/_monitoring-plugins/alerting/monitors.md b/_monitoring-plugins/alerting/monitors.md index c7f4284d..1b81fe99 100644 --- a/_monitoring-plugins/alerting/monitors.md +++ b/_monitoring-plugins/alerting/monitors.md @@ -103,7 +103,7 @@ POST _nodes/reload_secure_settings 1. Specify a name for the monitor. 1. Choose either **Per query monitor** or **Per bucket monitor**. -Whereas per-query monitors run your specified query and then check whether the query's results triggers any alerts, per-bucket monitors let you select fields to create buckets and categorize your results into those buckets. The alerting plugin runs each bucket's unique results against a script you define later, so you have finer control over which results should trigger alerts. Each of those buckets can trigger an alert, but per-query monitors can only trigger one alert at a time. +Whereas query-level monitors run your specified query and then check whether the query's results triggers any alerts, bucket-level monitors let you select fields to create buckets and categorize your results into those buckets. The alerting plugin runs each bucket's unique results against a script you define later, so you have finer control over which results should trigger alerts. Each of those buckets can trigger an alert, but query-level monitors can only trigger one alert at a time. 1. Define the monitor in one of three ways: visually, using a query, or using an anomaly detector. @@ -158,7 +158,7 @@ Whereas per-query monitors run your specified query and then check whether the q "Start" and "end" refer to the interval at which the monitor runs. See [Available variables](#available-variables). - To define a monitor visually, choose **Visual editor**. Then choose a source index, a timeframe, an aggregation (for example, `count()` or `average()`), a data filter if you want to monitor a subset of your source index, and a group-by field if you want to include an aggregation field in your query. Visual definition works well for most monitors. + To define a monitor visually, choose **Visual editor**. Then choose a source index, a timeframe, an aggregation (for example, `count()` or `average()`), a data filter if you want to monitor a subset of your source index, and a group-by field if you want to include an aggregation field in your query. At least one group-by field is required if you're defining a bucket-level monitor. Visual definition works well for most monitors. If you use the security plugin, you can only choose indices that you have permission to access. For details, see [Alerting security]({{site.url}}{{site.baseurl}}/security-plugin/). @@ -193,18 +193,20 @@ Steps to create a trigger differ depending on whether you chose **Visual editor* You begin by specifying a name and severity level for the trigger. Severity levels help you manage alerts. A trigger with a high severity level (e.g. 1) might page a specific individual, whereas a trigger with a low severity level might message a chat room. -Remember that per-query monitors run your trigger's script just once against the query's results, but per-bucket monitors execute your trigger's script on each bucket, so you should create a trigger that best fits the monitor you chose. If you want to execute multiple scripts, you must create multiple triggers. +Remember that query-level monitors run your trigger's script just once against the query's results, but bucket-level monitors execute your trigger's script on each bucket, so you should create a trigger that best fits the monitor you chose. If you want to execute multiple scripts, you must create multiple triggers. ### Visual editor -For **Trigger condition**, specify a threshold for the aggregation and timeframe you chose earlier, such as "is below 1,000" or "is exactly 10." +For a query-level monitor's **Trigger condition**, specify a threshold for the aggregation and timeframe you chose earlier, such as "is below 1,000" or "is exactly 10." The line moves up and down as you increase and decrease the threshold. Once this line is crossed, the trigger evaluates to true. +Bucket-level monitors also require you to specify a threshold and value for your aggregation and timeframe, but you can use a maximum of five conditions to better refine your trigger. Optionally, you can also use a keyword filter to filter for a specific field in your index. + ### Extraction query -For **Trigger condition**, specify a Painless script that returns true or false. Painless is the default OpenSearch scripting language and has a syntax similar to Groovy. +If you're using a query-level monitor, specify a Painless script that returns true or false. Painless is the default OpenSearch scripting language and has a syntax similar to Groovy. Trigger condition scripts revolve around the `ctx.results[0]` variable, which corresponds to the extraction query response. For example, your script might reference `ctx.results[0].hits.total.value` or `ctx.results[0].hits.hits[i]._source.error_code`. @@ -213,6 +215,27 @@ A return value of true means the trigger condition has been met, and the trigger The **Info** link next to **Trigger condition** contains a useful summary of the variables and results available to your query. {: .tip } +Bucket-level monitors require you to specify more information in your trigger condition. At a minimum, you must have the following fields: + +- `buckets_path`, which maps variable names to metrics to use in your script. +- `parent_bucket_path`, which is a path to a multi-bucket aggregation. The path can include single-bucket aggregations, but the last aggregation must be multi-bucket. For example, if you have a pipeline such as `agg1>agg2>agg3`, `agg1` and `agg2` are single-bucket aggregations, but `agg3` must be a multi-bucket aggregation. +- `script`, which is the script that OpenSearch runs to evaluate whether to trigger any alerts. + +For example, you might have a script that looks like the following: + +```json +{ + "buckets_path": { + "count_var": "_count" + }, + "parent_bucket_path": "composite_agg", + "script": { + "source": "params.count_var > 5" + } +} +``` + +After mapping the `count_var` variable to the `_count` metric, you can use `count_var` in your script and reference `_count` data. Finally, `composite_agg` is a path to a multi-bucket aggregation. ### Anomaly detector @@ -314,7 +337,7 @@ Variable | Data Type | Description `ctx.periodStart` | String | Unix timestamp for the beginning of the period during which the alert triggered. For example, if a monitor runs every ten minutes, a period might begin at 10:40 and end at 10:50. `ctx.periodEnd` | String | The end of the period during which the alert triggered. `ctx.error` | String | The error message if the trigger was unable to retrieve results or unable to evaluate the trigger, typically due to a compile error or null pointer exception. Null otherwise. -`ctx.alert` | Object | The current, active alert (if it exists). Includes `ctx.alert.id`, `ctx.alert.version`, and `ctx.alert.isAcknowledged`. Null if no alert is active. Only available with per-query monitors. +`ctx.alert` | Object | The current, active alert (if it exists). Includes `ctx.alert.id`, `ctx.alert.version`, and `ctx.alert.isAcknowledged`. Null if no alert is active. Only available with query-level monitors. `ctx.dedupedAlerts` | Object | Alerts that have already been triggered. OpenSearch keeps the existing alert to prevent the plugin from creating endless amounts of the same alerts. Only available with bucket-level monitors. `ctx.newAlerts` | Object | Newly created alerts. Only available with bucket-level monitors. `ctx.completedAlerts` | Object | Alerts that are no longer ongoing. Only available with bucket-level monitors. @@ -345,6 +368,7 @@ If you don't want to receive notifications for alerts, you don't have to add act ``` In this case, the message content must conform to the `Content-Type` header in the [custom webhook](#create-destinations). +1. If you're using a bucket-level monitor, you can choose whether the monitor should perform an action for each execution or for each alert. 1. (Optional) Use action throttling to limit the number of notifications you receive within a given span of time. From d4342aab59f119706f44ef88d0a8a5638331daa4 Mon Sep 17 00:00:00 2001 From: aetter Date: Thu, 30 Sep 2021 13:43:24 -0700 Subject: [PATCH 038/229] Initial 1.1.0 updates --- README.md | 8 ++++---- _config.yml | 6 +++--- _opensearch/install/plugins.md | 18 ++++++++++++++++++ _opensearch/install/tar.md | 5 +++-- 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 093941af..a9bd14e2 100644 --- a/README.md +++ b/README.md @@ -196,17 +196,17 @@ If you're making major changes to the documentation and need to see the rendered ## New releases 1. Branch. -1. Change the `opensearch_version` and `opensearch_major_minor_version` variables in `_config.yml`. +1. Change the `opensearch_version`, `opensearch_major_minor_version`, and `lucene_version` variables in `_config.yml`. 1. Start up a new cluster using the updated Docker Compose file in `docs/install/docker.md`. 1. Update the version table in `version-history.md`. - Use `curl -XGET https://localhost:9200 -u admin:admin -k` to verify the OpenSearch version. + Use `curl -XGET https://localhost:9200 -u admin:admin -k` to verify the OpenSearch and Lucene versions. -1. Update the plugin compatibility table in `docs/install/plugin.md`. +1. Update the plugin compatibility table in `_opensearch/install/plugin.md`. Use `curl -XGET https://localhost:9200/_cat/plugins -u admin:admin -k` to get the correct version strings. -1. Update the plugin compatibility table in `docs/opensearch-dashboards/plugins.md`. +1. Update the plugin compatibility table in `_dashboards/install/plugins.md`. Use `docker ps` to find the ID for the OpenSearch Dashboards node. Then use `docker exec -it /bin/bash` to get shell access. Finally, run `./bin/opensearch-dashboards-plugin list` to get the plugins and version strings. diff --git a/_config.yml b/_config.yml index ae992e6d..e22718a4 100644 --- a/_config.yml +++ b/_config.yml @@ -5,9 +5,9 @@ baseurl: "/docs" # the subpath of your site, e.g. /blog url: "https://opensearch.org" # the base hostname & protocol for your site, e.g. http://example.com permalink: /:path/ -opensearch_version: 1.0.1 -opensearch_major_minor_version: 1.0 -lucene_version: 8_8_2 +opensearch_version: 1.1.0 +opensearch_major_minor_version: 1.1 +lucene_version: 8_9_0 # Build settings markdown: kramdown diff --git a/_opensearch/install/plugins.md b/_opensearch/install/plugins.md index 40fdefc7..bd0d2d01 100644 --- a/_opensearch/install/plugins.md +++ b/_opensearch/install/plugins.md @@ -29,6 +29,24 @@ If you don't want to use the all-in-one OpenSearch installation options, you can + + 1.1.0 + +
opensearch-alerting                  1.1.0.0
+opensearch-anomaly-detection         1.1.0.0
+opensearch-asynchronous-search       1.1.0.0
+opensearch-cross-cluster-replication 1.1.0.0
+opensearch-index-management          1.1.0.0
+opensearch-job-scheduler             1.1.0.0
+opensearch-knn                       1.1.0.0
+opensearch-notebooks                 1.1.0.0
+opensearch-performance-analyzer      1.1.0.0
+opensearch-reports-scheduler         1.1.0.0
+opensearch-security                  1.1.0.0
+opensearch-sql                       1.1.0.0
+
+ + 1.0.1 diff --git a/_opensearch/install/tar.md b/_opensearch/install/tar.md index af45a3af..70151f5f 100644 --- a/_opensearch/install/tar.md +++ b/_opensearch/install/tar.md @@ -18,9 +18,10 @@ The tarball supports most Linux distributions, including CentOS 7, Amazon Linux ```bash # x64 tar -zxf opensearch-{{site.opensearch_version}}-linux-x64.tar.gz - cd opensearch-{{site.opensearch_version}}{% comment %}# ARM64 + cd opensearch-{{site.opensearch_version}} + # ARM64 tar -zxf opensearch-{{site.opensearch_version}}-linux-arm64.tar.gz - cd opensearch-{{site.opensearch_version}}{% endcomment %} + cd opensearch-{{site.opensearch_version}} ``` 1. Run OpenSearch: From ab00a055490e09d13d5b45c5a5e682e8465f1002 Mon Sep 17 00:00:00 2001 From: aetter Date: Thu, 30 Sep 2021 13:48:32 -0700 Subject: [PATCH 039/229] Comment out Java for now --- _clients/index.md | 2 ++ _clients/python.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/_clients/index.md b/_clients/index.md index d920e16a..2f3513dd 100644 --- a/_clients/index.md +++ b/_clients/index.md @@ -13,7 +13,9 @@ OpenSearch provides clients for several popular programming languages, with more For example, a 1.0.0 client works with an OpenSearch 1.1.0 cluster, but might not support any non-breaking API changes in OpenSearch 1.1.0. A 1.2.0 client works with the same cluster, but might allow you to pass unsupported options in certain functions. We recommend using the same version for both, but if your tests pass after a cluster upgrade, you don't necessarily need to upgrade your clients immediately. +{% comment %} * [OpenSearch Java client]({{site.url}}{{site.baseurl}}/clients/java/) +{% endcomment %} * [OpenSearch Python client]({{site.url}}{{site.baseurl}}/clients/python/) * [OpenSearch JavaScript (Node.js) client]({{site.url}}{{site.baseurl}}/clients/javascript/) * [OpenSearch Go client]({{site.url}}{{site.baseurl}}/clients/go/) diff --git a/_clients/python.md b/_clients/python.md index 9ce68bca..10a856a2 100644 --- a/_clients/python.md +++ b/_clients/python.md @@ -6,7 +6,7 @@ nav_order: 70 # Python client -The OpenSearch Python client provides a more natural syntax for interacting with your cluster. Rather than sending HTTP requests with raw JSON bodies to a given URL, you can create an OpenSearch client for your cluster and call the client's built-in functions. +The OpenSearch Python client provides a more natural syntax for interacting with your cluster. Rather than sending HTTP requests to a given URL, you can create an OpenSearch client for your cluster and call the client's built-in functions. {% comment %} `opensearch-py` is the lower-level of the two Python clients. If you want a general client for assorted operations, it's a great choice. If you want a higher-level client strictly for indexing and search operations, consider [opensearch-dsl-py]({{site.url}}{{site.baseurl}}/clients/python-dsl/). From 7cb714894fba8d1401dbab84824384b226cea616 Mon Sep 17 00:00:00 2001 From: aetter Date: Thu, 30 Sep 2021 13:52:34 -0700 Subject: [PATCH 040/229] Nitpicks --- _clients/javascript.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/_clients/javascript.md b/_clients/javascript.md index cb308a29..c670e4b8 100644 --- a/_clients/javascript.md +++ b/_clients/javascript.md @@ -1,18 +1,18 @@ --- layout: default -title: Javascript client +title: JavaScript client nav_order: 90 --- -# Javascript client +# JavaScript client -The OpenSearch Javascript client provides a safer and easier way to interact with your OpenSearch cluster. Rather than using OpenSearch from the browser and potentially exposing your data to the public, you can build an OpenSearch client that takes care of sending requests to your cluster. +The OpenSearch JavaScript client provides a safer and easier way to interact with your OpenSearch cluster. Rather than using OpenSearch from the browser and potentially exposing your data to the public, you can build an OpenSearch client that takes care of sending requests to your cluster. The client contains a library of APIs that let you perform different operations on your cluster and return a standard response body. The example here demonstrates some basic operations like creating an index, adding documents, and searching your data. ## Setup -To add the client to your project, install it from npm: +To add the client to your project, install it from [npm](https://www.npmjs.com): ```bash npm install @opensearch-project/opensearch From c378da8799fa3c10e85dc42905c859cf902e28f7 Mon Sep 17 00:00:00 2001 From: aetter Date: Thu, 30 Sep 2021 14:01:14 -0700 Subject: [PATCH 041/229] Make name more explicit --- _clients/java-rest-high-level.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/_clients/java-rest-high-level.md b/_clients/java-rest-high-level.md index def97086..892f7341 100644 --- a/_clients/java-rest-high-level.md +++ b/_clients/java-rest-high-level.md @@ -1,10 +1,10 @@ --- layout: default -title: Java high-level REST client +title: Elasticsearch OSS Java high-level REST client nav_order: 60 --- -# Java high-level REST client +# Elasticsearch OSS Java high-level REST client The Elasticsearch OSS Java high-level REST client allows you to interact with your OpenSearch clusters and indices through Java methods and data structures rather than HTTP methods and JSON. @@ -22,7 +22,7 @@ To start using the Elasticsearch OSS Java high-level REST client, ensure that yo ``` -You can now start your OpenSearch cluster. The 7.10.2 high-level REST client works with the 1.x versions of OpenSearch. +You can now start your OpenSearch cluster. The 7.10.2 Elasticsearch OSS high-level REST client works with the 1.x versions of OpenSearch. ## Sample code From 5862b1b300fda22885b9fb89745330cd929e7ad6 Mon Sep 17 00:00:00 2001 From: Liz Snyder Date: Thu, 30 Sep 2021 16:10:28 -0700 Subject: [PATCH 042/229] First crack at CCR docs --- _config.yml | 6 + _replication-plugin/api.md | 243 ++++++++++++++++ _replication-plugin/auto-follow.md | 76 +++++ _replication-plugin/get-started.md | 271 ++++++++++++++++++ _replication-plugin/index.md | 19 ++ _replication-plugin/permissions.md | 78 +++++ _replication-plugin/settings.md | 32 +++ .../access-control/permissions.md | 11 + .../access-control/users-roles.md | 2 + 9 files changed, 738 insertions(+) create mode 100644 _replication-plugin/api.md create mode 100644 _replication-plugin/auto-follow.md create mode 100644 _replication-plugin/get-started.md create mode 100644 _replication-plugin/index.md create mode 100644 _replication-plugin/permissions.md create mode 100644 _replication-plugin/settings.md diff --git a/_config.yml b/_config.yml index ae992e6d..22daf67d 100644 --- a/_config.yml +++ b/_config.yml @@ -45,6 +45,9 @@ collections: im-plugin: permalink: /:collection/:path/ output: true + replication-plugin: + permalink: /:collection/:path/ + output: true monitoring-plugins: permalink: /:collection/:path/ output: true @@ -81,6 +84,9 @@ just_the_docs: im-plugin: name: Index management plugin nav_fold: true + replication-plugin: + name: Replication plugin + nav_fold: true monitoring-plugins: name: Monitoring plugins nav_fold: true diff --git a/_replication-plugin/api.md b/_replication-plugin/api.md new file mode 100644 index 00000000..a552d1c2 --- /dev/null +++ b/_replication-plugin/api.md @@ -0,0 +1,243 @@ +--- +layout: default +title: API +nav_order: 50 +--- + +# Cross-cluster replication API + +Use these replication operations to programmatically manage cross-cluster replication. + +#### Table of contents +- TOC +{:toc} + +## Start replication +Introduced 1.1 +{: .label .label-purple } + +Initiate replication of an index from the leader cluster to the follower cluster. Run this operation on the follower cluster. + + +#### Request + +```json +PUT /_plugins/_replication//_start +{ + "leader_alias":"", + "leader_index":"", + "use_roles":{ + "leader_cluster_role":"", + "follower_cluster_role":"" + } +} +``` + +Specify the following options: + +Options | Description | Type | Required +:--- | :--- |:--- |:--- | +`leader_alias` | The name of the leader cluster. | `string` | Yes +`leader_index` | The index on the leader cluster that you want to replicate. | `string` | Yes +`use_roles` | The roles to use for all subsequent backend replication tasks between the indices. Specify a `leader_cluster_role` and `follower_cluster_role`. See [Map the leader and follower cluster roles]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). | `string` | If security plugin is enabled + +#### Sample response + +```json +{ + "acknowledged": true +} +``` + +## Stop replication +Introduced 1.1 +{: .label .label-purple } + +Terminates replication and converts the follower index to a standard index. + +#### Request + +```json +POST /_plugins/_replication//_stop +``` + +#### Sample response + +```json +{ + "acknowledged": true +} +``` + +## Pause replication +Introduced 1.1 +{: .label .label-purple } + +Pauses replication of the leader index. If you don't resume replication after 12 hours, it stops completely and the follower index is converted to a standard index. + +#### Request + +```json +PUT /_plugins/_replication//_pause +``` + +#### Sample response + +```json +{ + "acknowledged": true +} +``` + +## Resume replication +Introduced 1.1 +{: .label .label-purple } + +Resumes replication of the leader index. + +#### Request + +```json +PUT /_plugins/_replication//_resume +``` + +#### Sample response + +```json +{ + "acknowledged": true +} +``` + +## Get replication status +Introduced 1.1 +{: .label .label-purple } + +Gets the status of index replication. You can use this API to measure replication lag. Run this command from the leader cluster. + +#### Request + +```json +GET /_plugins/_replication//_status +``` + +#### Sample response + +```json +{ + "status":"SYNCING", + "reason":"User initiated", + "remote_cluster":"remote-cluster", + "leader_index":"leader-01", + "follower_index":"follower-01", + "syncing_details":{ + "remote_checkpoint": 19, + "local_checkpoint": 19, + "seq_no": 20 + } +} +``` + +To include shard replication details in the response, add `&verbose=true`. + +## Update settings +Introduced 1.1 +{: .label .label-purple } + +Updates settings on the follower index. + +#### Request + +```json +PUT /_plugins/_replication//_update +{ + "settings":{ + "index.number_of_shards": 4, + "index.number_of_replicas": 2 + } +} +``` + +#### Sample response + +```json +{ + "acknowledged": true +} +``` + +## Create replication rule +Introduced 1.1 +{: .label .label-purple } + +Automatically starts replication on indices matching a specified pattern. Newly created indices on the remote cluster that match one of the specified patterns will be automatically configured as follower indices. You can also use this API to update existing auto-follow patterns. + +Run this command on the follower cluster. + +Make sure to note the names of all auto-follow patterns after you create them. The replication plugin currently does not include an API operation to retrieve a list of existing patterns. +{: .tip } + +#### Request + +```json +POST /_plugins/_replication/_autofollow +{ + "leader_alias" : "", + "name": "", + "pattern": "", + "use_roles":{ + "leader_cluster_role": "", + "follower_cluster_role": "" + } +} +``` + +Specify the following options: + +Options | Description | Type | Required +:--- | :--- |:--- |:--- | +`leader_alias` | The name of the remote cluster to associate the pattern with. | `string` | Yes +`name` | A name for the auto-follow pattern. | `string` | Yes +`pattern` | An array of index patterns to match against indices in the specified leader cluster. Supports wildcard characters. For example, `leader-*`. | `string` | Yes +`use_roles` | The roles to use for all subsequent backend replication tasks between the indices. Specify a `leader_cluster_role` and `follower_cluster_role`. See [Map the leader and follower cluster roles]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). | `string` | If security plugin is enabled + +#### Sample response + +```json +{ + "acknowledged": true +} +``` + +## Delete replication rule +Introduced 1.1 +{: .label .label-purple } + +Deletes the specified replication rule. This operation prevents any new indices from being replicated but does not stop existing replication that the rule has already initiated. + +Run this command on the follower cluster. + +#### Request + +```json +DELETE /_plugins/_replication/_autofollow +{ + "leader_alias" : "", + "name": "", +} +``` + +Specify the following options: + +Options | Description | Type | Required +:--- | :--- |:--- |:--- | +`leader_alias` | The name of the remote cluster that the pattern is associated with. | `string` | Yes +`name` | The name of the pattern. | `string` | Yes + +#### Sample response + +```json +{ + "acknowledged": true +} +``` diff --git a/_replication-plugin/auto-follow.md b/_replication-plugin/auto-follow.md new file mode 100644 index 00000000..2af9551c --- /dev/null +++ b/_replication-plugin/auto-follow.md @@ -0,0 +1,76 @@ +--- +layout: default +title: Auto-follow +nav_order: 20 +has_children: false + +--- + +# Auto-follow for cross-cluster replication + +Auto-follow lets you automatically replicate indices created on the leader cluster based on matching patterns. When you create an index on the leader cluster with a name that matches a specified pattern (for example, `index-01*`), a corresponding follower index is automatically created on the follower cluster. + +You can configure multiple replication rules for a single cluster. The patterns currently only support wildcard matching. + +## Prerequisites + +You need to [set up a cross-cluster connection]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/#set-up-a-cross-cluster-connection) between two clusters before you can enable auto-follow. + +## Permissions + +If the security plugin is enabled, non-admin users need to be mapped to the appropriate permissions in order to perform replication actions. For index and cluster-level permissions requirements, see [Cross-cluster replication permissions]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/). + +## Get started with auto-follow + +Replication rules are a collection of patterns that you create against a single remote cluster. When you create a replication rule, it automatically starts replicating any *new* indices that match the pattern, but does not replicate matching indices that were previously created. + +Make sure to note the names of all rules when you create them. The replication plugin currently does not include an API operation to retrieve a list of existing rules. +{: .tip } + +Create a replication rule on the follower cluster: + +```bash +curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/_autofollow?pretty' -d ' +{ + "leader_alias" : "leader-cluster", + "name": "my-replication-rule", + "pattern": "movies*", + "use_roles":{ + "leader_cluster_role": "all_access", + "follower_cluster_role": "all_access" + } +}' +``` + +If the security plugin is disabled, you can leave out the `use_roles` parameter. If it's enabled, however, you need to specify the leader and follower cluster roles that OpenSearch will use to authenticate the request. This example uses `all_access` for simplicity, but we recommend creating a replication user on each cluster and [mapping it accordingly]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). +{: .tip } + +To test the rule, create a matching index on the leader cluster: + +```bash +curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/movies-0001' +``` + +And confirm its replica shows up on the follower cluster: + +```bash +curl -XGET -u 'admin:admin' -k 'https://localhost:9200/_cat/indices?v' +``` + +```bash +health status index uuid pri rep docs.count docs.deleted store.size pri.store.size +yellow open movies-0001 kHOxYYHxRMeszLjTD9rvSQ 1 1 0 0 208b 208b +``` + +## Delete a replication rule + +When you delete a replication rule, OpenSearch stops replicating *new* indices that match the pattern, but replication of existing indices that the rule previously created will continue. If you need to stop existing replication activity, use the [stop replication API operation]({{site.url}}{{site.baseurl}}/replication-plugin/api/#stop-replication). + +```bash +curl -XDELETE -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/_autofollow?pretty' -d ' +{ + "leader_alias" : "leader-cluster", + "name": "my-replication-rule" +}' +``` + diff --git a/_replication-plugin/get-started.md b/_replication-plugin/get-started.md new file mode 100644 index 00000000..00576fa4 --- /dev/null +++ b/_replication-plugin/get-started.md @@ -0,0 +1,271 @@ +--- +layout: default +title: Get started +nav_order: 10 +--- + +# Get started with cross-cluster replication + +With cross-cluster replication, you index data to a leader index and that data is replicated to one or more read-only follower indices. All subsequnt operations on the leader are replicated on the follower, such as creating, updating, or deleting documents. + +## Prerequisites + +Cross-cluster replication has the following prerequisites: +- Install the replication plugin on all nodes of both the leader and the follower cluster. +- If you've overridden `node.roles` in opensearch.yml on the remote cluster, make sure it also includes the `remote_cluster_client` role: + + ```yaml + node.roles: [, remote_cluster_client] + ``` + +## Permissions + +Make sure the security plugin is either enabled on both clusters or disabled on both clusters. If you disabled the security plugin, you can skip this section. + +If the security plugin is enabled, non-admin users need to be mapped to the appropriate permissions in order to perform replication actions. For index and cluster-level permissions requirements, see [Cross-cluster replication permissions]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/). + +In addition, add the following setting to opensearch.yml on the leader cluster so it allows connections from the follower cluster: + +```yml +plugins.security.nodes_dn_dynamic_config_enabled: true +``` + +## Example setup + +Save this sample file as `docker-compose.yml` and run `docker-compose up` to start two single-node clusters on the same network: + +```yml +version: '3' +services: + replication-node1: + image: opensearchproject/opensearch:{{site.opensearch_version}} + container_name: replication-node1 + environment: + - cluster.name=leader-cluster + - discovery.type=single-node + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - opensearch-data2:/usr/share/opensearch/data + ports: + - 9201:9200 + - 9700:9600 # required for Performance Analyzer + networks: + - opensearch-net + replication-node2: + image: opensearchproject/opensearch:{{site.opensearch_version}} + container_name: replication-node2 + environment: + - cluster.name=follower-cluster + - discovery.type=single-node + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - opensearch-data1:/usr/share/opensearch/data + ports: + - 9200:9200 + - 9600:9600 # required for Performance Analyzer + networks: + - opensearch-net + +volumes: + opensearch-data1: + opensearch-data2: + +networks: + opensearch-net: +``` + +After the clusters start, verify the names of each: + +```bash +curl -XGET -u 'admin:admin' -k 'https://localhost:9201' +{ + "name" : "replication-node1", + "cluster_name" : "leader-cluster", + ... +} + +curl -XGET -u 'admin:admin' -k 'https://localhost:9200' +{ + "name" : "replication-node2", + "cluster_name" : "follower-cluster", + ... +} +``` + +For this example, use port 9201 (`replication-node1`) as the leader and port 9200 (`replication-node2`) as the follower cluster. + +To get the IP address for the leader cluster, first identify its container ID: + +```bash +docker ps +CONTAINER ID IMAGE PORTS NAMES +3b8cdc698be5 opensearchproject/opensearch:{{site.opensearch_version}} 0.0.0.0:9200->9200/tcp, 0.0.0.0:9600->9600/tcp, 9300/tcp replication-node1 +731f5e8b0f4b opensearchproject/opensearch:{{site.opensearch_version}} 9300/tcp, 0.0.0.0:9201->9200/tcp, 0.0.0.0:9700->9600/tcp replication-node2 +``` + +Then get that container's IP address: + +```bash +docker inspect --format='{% raw %}{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}{% endraw %}' 731f5e8b0f4b +172.22.0.3 +``` + +## Set up a cross-cluster connection + +On the follower cluster, add the leader cluster name and the IP address (with port 9300) for each seed node. In this case, you only have one seed node: + +```bash +curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_cluster/settings?pretty' -d ' +{ + "persistent": { + "cluster": { + "remote": { + "leader-cluster": { + "seeds": ["172.22.0.3:9300"] + } + } + } + } +}' +``` + +## Start replication + +To get started, create an index called `leader-01` on the remote (leader) cluster: + +```bash +curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/leader-01?pretty' +``` + +Start replication of that index from the follower cluster. Starting replication creates the provided follower index from scratch; you can't convert an existing index to a follower index. + +Provide the leader cluster and index that you want to replicate: + +```bash +curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_start?pretty' -d ' +{ + "leader_alias": "leader-cluster", + "leader_index": "leader-01", + "use_roles":{ + "leader_cluster_role": "all_access", + "follower_cluster_role": "all_access" + } +}' +``` + +If the security plugin is disabled, you can leave out the `use_roles` parameter. If it's enabled, however, you need to specify the leader and follower cluster roles that OpenSearch will use to authenticate the request. This example uses `all_access` for simplicity, but we recommend creating a replication user on each cluster and [mapping it accordingly]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). +{: .tip } + +This command creates an identical read-only index named "follower-01" on the local cluster that continuously stays updated with changes to the "leader-01" index on the remote cluster. + +After replication starts, get the status: + +```bash +curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_status?pretty' + +{ + "status" : "SYNCING", + "reason" : "User initiated", + "leader_alias" : "leader-cluster", + "leader_index" : "leader-01", + "follower_index" : "follower-01", + "syncing_details" : { + "leader_checkpoint" : -1, + "follower_checkpoint" : -1, + "seq_no" : 0 + } +} +``` + +## Confirm replication + +To confirm that replication is actually happening, add a document to the leader index: + +```bash +curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/leader-01/_doc/1?pretty' -d '{"The Shining": "Stephen King"}' +``` + +Then validate the replicated content on the follower index: + +```bash +curl -XGET -k -u 'admin:admin' 'https://localhost:9200/follower-01/_search?pretty' + +{ + ... + "hits": [{ + "_index": "follower-01", + "_type": "_doc", + "_id": "1", + "_score": 1.0, + "_source": { + "The Shining": "Stephen King" + } + }] +} +``` + +## Pause and resume replication + +You can temporarily pause replication of an index if you need to remediate issues or reduce load on the leader cluster: + +```bash +curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_pause?pretty' -d '{}' +``` + +To confirm replication is paused, get the status: + +```bash +curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_status?pretty' + +{ + "status" : "PAUSED", + "reason" : "User initiated", + "leader_alias" : "leader-cluster", + "leader_index" : "leader-01", + "follower_index" : "follower-01" +} +``` + +When you're done making changes, resume replication: + +```bash +curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_resume?pretty' -d '{}' +``` + +When replication resumes, the follower index picks up any changes that were made to the leader index while replication was paused. + +If you don't resume replication within 12 hours, replication stops completely and the follower index is converted to a standard index. + +## Stop replication + +Terminate replication of a specified index from the follower cluster: + +```bash +curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_stop' -d '{}' +``` + +When you stop replication, the follower index un-follows the leader and becomes a standard index that you can write to. You can't restart replication after it's been terminated. + +Get the status to confirm that the index is no longer being replicated: + +```bash +curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_status?pretty' + +{ + "status" : "REPLICATION NOT IN PROGRESS" +} +``` + +You can further confirm that replication is stopped by making modifications to the leader index and confirming they don't show up on the follower index. + + diff --git a/_replication-plugin/index.md b/_replication-plugin/index.md new file mode 100644 index 00000000..1b9414f2 --- /dev/null +++ b/_replication-plugin/index.md @@ -0,0 +1,19 @@ +--- +layout: default +title: Cross-cluster replication +nav_order: 1 +has_children: false + +--- + +# Cross-cluster replication + +The cross-cluster replication plugin lets you replicate indices, mappings, and metadata from one OpenSearch cluster to another. It follows an active-passive replication model where the follower index (where the data is replicated) pulls data from the leader (source) index. + +The replication plugin supports replication of indices using wildcard pattern matching and provides commands to pause, resume, and stop replication. Once replication starts on an index, it initiates a persistent background task on the primary shard of the follower cluster that continuously polls corresponding shards from the leader cluster for updates. + +The replication plugin integrates with the security plugin so you can encrypt cross-cluster traffic with node-to-node encryption and control access to replication activities. + +To start, see [Get started with cross-cluster replication]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/). + + diff --git a/_replication-plugin/permissions.md b/_replication-plugin/permissions.md new file mode 100644 index 00000000..6b3804ad --- /dev/null +++ b/_replication-plugin/permissions.md @@ -0,0 +1,78 @@ +--- +layout: default +title: Permissions +nav_order: 30 +--- + +# Cross-cluster replication permissions + +You can use the [security plugin]({{site.url}}{{site.baseurl}}/security-plugin/index/) with cross-cluster replication to limit users to certain actions. For example, you might want certain users to only perform replication activity on the leader or follower cluster. + +Because cross-cluster replication involves multiple clusters, it's possible that clusters might have different security configurations. The following configurations are supported: + +- Security plugin fully enabled on both clusters +- Security plugin enabled only for TLS on both clusters (`plugins.security.ssl_only`) +- Security plugin absent or disabled on both clusters (not recommended) + +You can enable node-to-node encryption on both the leader and the follower cluster to ensure that replication traffic between the clusters is encrypted. + +## Basic permissions + +In order for non-admin users to perform replication activities, they need to be mapped to the appropriate permissions. + +The security plugin has two built-in roles that cover most replication use cases: `cross_cluster_replication_leader_full_access` which provides replication permissions on the leader cluster, and `cross_cluster_replication_follower_full_access` which provides replication permissions on the follower cluster. For descriptions of each, see [Predefined roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/users-roles#predefined-roles). + +If you don't want to use the default roles, you can combine individual replication [permissions]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#replication-permissions) to meet your needs. Most permissions correspond to specific REST API operations. For example, the `indices:admin/plugins/replication/index/pause` permission lets you pause replication. + +## Map the leader and follower cluster roles + +The [start replication]({{site.url}}{{site.baseurl}}/replication-plugin/api/#start-replication) and [create replication rule]({{site.url}}{{site.baseurl}}/replication-plugin/api/#start-replication) are special cases because they involve background processes that require permissions on both the leader and the follower cluster. Therefore, when you perform one of these actions, you need to explicitly pass the `leader_cluster_role` and +`follower_cluster_role` in the request, which OpenSearch will then use in all backend replication tasks. + +To enable non-admins to start replication and create replication rules, create an identical user on each cluster (for example, `replication_user`) and map them to the `cross_cluster_replication_leader_full_access` role on the remote cluster and `cross_cluster_replication_follower_full_access` on the follower cluster. For instructions, see [Map users to roles]({{site.url}}{{site.baseurl}}/access-control/users-roles/#map-users-to-roles). + +You can then pass those roles into the request along with the appropriate credentials: + +```bash +curl -XPUT -k -H 'Content-Type: application/json' -u 'replication_user:password' 'https://localhost:9200/_plugins/_replication/follower-01/_start?pretty' -d ' +{ + "leader_alias": "leader-cluster", + "leader_index": "leader-01", + "use_roles":{ + "leader_cluster_role": "cross_cluster_replication_leader_full_access", + "follower_cluster_role": "cross_cluster_replication_follower_full_access" + } +}' +``` + +You can instead choose to create your own leader and follower cluster roles to meet your needs, but we recommend using the default roles. + +## Replication permissions + +The following sections list the available index and cluster-level permissions for cross-cluster replication. + +### Follower cluster + +Users can have the following permissions for the follower cluster: + +``` +indices:admin/plugins/replication/index/setup/validate +indices:admin/plugins/replication/index/start +indices:admin/plugins/replication/index/pause +indices:admin/plugins/replication/index/resume +indices:admin/plugins/replication/index/stop +indices:admin/plugins/replication/index/update +indices:admin/plugins/replication/index/status_check +indices:data/write/plugins/replication/changes +cluster:admin/plugins/replication/autofollow/update +``` + +### Leader cluster + +Users can have the following permissions for the leader cluster: + +``` +indices:admin/plugins/replication/validate +indices:data/read/plugins/replication/file_chunk +indices:data/read/plugins/replication/changes +``` diff --git a/_replication-plugin/settings.md b/_replication-plugin/settings.md new file mode 100644 index 00000000..66d16ec4 --- /dev/null +++ b/_replication-plugin/settings.md @@ -0,0 +1,32 @@ +--- +layout: default +title: Settings +nav_order: 40 +--- + +# Replication settings + +The replication plugin adds several settings to the standard OpenSearch cluster settings. +The settings are dynamic, so you can change the default behavior of the plugin without restarting your cluster. +You can mark settings as `persistent` or `transient`. + +For example, to update the retention period of the result index: + +```json +PUT _cluster/settings +{ + "persistent": { + "plugins.replication.indices.recovery.parallel_chunks": "8" + } +} +``` + +These settings manage the resources consumed by remote recoveries. We don’t recommend changing these settings; the defaults should work well for most use cases. + +Setting | Default | Description +:--- | :--- | :--- +`plugins.replication.indices.recovery.chunk_size` | 1MB | The chunk size requested by the follower cluster during file transfer. Specify the chunk size as a value and unit, for example, 10MB, 5KB. +`plugins.replication.indices.recovery.parallel_chunks` | 5 | The number of file chunk requests that can be sent in parallel for each recovery. +`plugins.replication.indices.recovery.request_timeout` | 60s | The amount of time to wait for individual network requests during the remote recovery process. A single action timeout can cause recovery to fail. +`plugins.replication.indices.recovery.activity_timeout` | 5m | The amount of time to wait for recovery activity. If the leader cluster doesn't receive recovery requests from the follower for this amount of time, it closes the in-memory resources needed to supply data to the follower during recovery. + diff --git a/_security-plugin/access-control/permissions.md b/_security-plugin/access-control/permissions.md index 192e8a7b..44db6825 100644 --- a/_security-plugin/access-control/permissions.md +++ b/_security-plugin/access-control/permissions.md @@ -59,6 +59,7 @@ Rather than creating new action groups from individual permissions, you can ofte - cluster:admin/opensearch/reports/instance/get - cluster:admin/opensearch/reports/instance/list - cluster:admin/opensearch/reports/menu/download +- cluster:admin/plugins/replication/autofollow/update - cluster:admin/reindex/rethrottle - cluster:admin/repository/delete - cluster:admin/repository/get @@ -114,6 +115,13 @@ Rather than creating new action groups from individual permissions, you can ofte - indices:admin/mappings/fields/get* - indices:admin/mappings/get - indices:admin/open +- indices:admin/plugins/replication/index/setup/validate +- indices:admin/plugins/replication/index/start +- indices:admin/plugins/replication/index/pause +- indices:admin/plugins/replication/index/resume +- indices:admin/plugins/replication/index/stop +- indices:admin/plugins/replication/index/update +- indices:admin/plugins/replication/index/status_check - indices:admin/refresh - indices:admin/refresh* - indices:admin/resolve/index @@ -139,6 +147,8 @@ Rather than creating new action groups from individual permissions, you can ofte - indices:data/read/msearch/template - indices:data/read/mtv - indices:data/read/mtv* +- indices:data/read/plugins/replication/file_chunk +- indices:data/read/plugins/replication/changes - indices:data/read/scroll - indices:data/read/scroll/clear - indices:data/read/search @@ -150,6 +160,7 @@ Rather than creating new action groups from individual permissions, you can ofte - indices:data/write/delete - indices:data/write/delete/byquery - indices:data/write/index +- indices:data/write/plugins/replication/changes - indices:data/write/reindex - indices:data/write/update - indices:data/write/update/byquery diff --git a/_security-plugin/access-control/users-roles.md b/_security-plugin/access-control/users-roles.md index 445e1e63..b69cae4c 100644 --- a/_security-plugin/access-control/users-roles.md +++ b/_security-plugin/access-control/users-roles.md @@ -109,6 +109,8 @@ Role | Description `anomaly_full_access` | Grants full permissions to all anomaly detection actions. `anomaly_read_access` | Grants permissions to view detectors, but not create, modify, or delete detectors. `all_access` | Grants full access to the cluster: all cluster-wide operations, write to all indices, write to all tenants. +`cross_cluster_replication_follower_full_access` | Grants full access to perform cross-cluster replication actions on the follower cluster. +`cross_cluster_replication_leader_full_access` | Grants full access to perform cross-cluster replication actions on the leader cluster. `kibana_read_only` | A special role that prevents users from making changes to visualizations, dashboards, and other OpenSearch Dashboards objects. See `plugins.security.readonly_mode.roles` in `opensearch_dashboards.yml`. Pair with the `kibana_user` role. `kibana_user` | Grants permissions to use OpenSearch Dashboards: cluster-wide searches, index monitoring, and write to various OpenSearch Dashboards indices. `logstash` | Grants permissions for Logstash to interact with the cluster: cluster-wide searches, cluster monitoring, and write to the various Logstash indices. From 8d3ae42f6af39169afb4c400bdb1b795587bed50 Mon Sep 17 00:00:00 2001 From: Liz Snyder Date: Thu, 30 Sep 2021 16:43:05 -0700 Subject: [PATCH 043/229] Typo --- _replication-plugin/get-started.md | 4 ++-- _replication-plugin/index.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/_replication-plugin/get-started.md b/_replication-plugin/get-started.md index 00576fa4..34e2899a 100644 --- a/_replication-plugin/get-started.md +++ b/_replication-plugin/get-started.md @@ -6,7 +6,7 @@ nav_order: 10 # Get started with cross-cluster replication -With cross-cluster replication, you index data to a leader index and that data is replicated to one or more read-only follower indices. All subsequnt operations on the leader are replicated on the follower, such as creating, updating, or deleting documents. +With cross-cluster replication, you index data to a leader index and that data is replicated to one or more read-only follower indices. All subsequent operations on the leader are replicated on the follower, such as creating, updating, or deleting documents. ## Prerequisites @@ -147,7 +147,7 @@ To get started, create an index called `leader-01` on the remote (leader) cluste curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/leader-01?pretty' ``` -Start replication of that index from the follower cluster. Starting replication creates the provided follower index from scratch; you can't convert an existing index to a follower index. +Start replication of that index from the follower cluster. Starting replication creates a follower index from scratch; you can't convert an existing index to a follower index. Provide the leader cluster and index that you want to replicate: diff --git a/_replication-plugin/index.md b/_replication-plugin/index.md index 1b9414f2..07ba256f 100644 --- a/_replication-plugin/index.md +++ b/_replication-plugin/index.md @@ -10,7 +10,7 @@ has_children: false The cross-cluster replication plugin lets you replicate indices, mappings, and metadata from one OpenSearch cluster to another. It follows an active-passive replication model where the follower index (where the data is replicated) pulls data from the leader (source) index. -The replication plugin supports replication of indices using wildcard pattern matching and provides commands to pause, resume, and stop replication. Once replication starts on an index, it initiates a persistent background task on the primary shard of the follower cluster that continuously polls corresponding shards from the leader cluster for updates. +The replication plugin supports replication of indices using wildcard pattern matching and provides commands to pause, resume, and stop replication. Once replication starts on an index, it initiates a persistent background task on the primary shard on the follower cluster that continuously polls corresponding shards from the leader cluster for updates. The replication plugin integrates with the security plugin so you can encrypt cross-cluster traffic with node-to-node encryption and control access to replication activities. From 25c4b70046dcd5016454678e3757390f3229e4d5 Mon Sep 17 00:00:00 2001 From: aetter Date: Thu, 30 Sep 2021 19:58:37 -0700 Subject: [PATCH 044/229] Adds plugin list and ARM tarball --- _dashboards/install/plugins.md | 15 +++++++++++++++ _dashboards/install/tar.md | 7 +++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/_dashboards/install/plugins.md b/_dashboards/install/plugins.md index 06b3c4c9..e0fc9d29 100644 --- a/_dashboards/install/plugins.md +++ b/_dashboards/install/plugins.md @@ -28,6 +28,21 @@ If you don't want to use the all-in-one installation options, you can install th + + 1.1.0 + +
alertingDashboards          1.1.0.0
+anomalyDetectionDashboards  1.1.0.0
+ganttChartDashboards        1.1.0.0
+indexManagementDashboards   1.1.0.0
+notebooksDashboards         1.1.0.0
+queryWorkbenchDashboards    1.1.0.0
+reportsDashboards           1.1.0.0
+securityDashboards          1.1.0.0
+traceAnalyticsDashboards    1.1.0.0
+
+ + 1.0.1 diff --git a/_dashboards/install/tar.md b/_dashboards/install/tar.md index 1c7e6933..026f23f7 100644 --- a/_dashboards/install/tar.md +++ b/_dashboards/install/tar.md @@ -14,9 +14,10 @@ nav_order: 30 ```bash # x64 tar -zxf opensearch-dashboards-{{site.opensearch_version}}-linux-x64.tar.gz - cd opensearch-dashboards{% comment %}# ARM64 + cd opensearch-dashboards + # ARM64 tar -zxf opensearch-dashboards-{{site.opensearch_version}}-linux-arm64.tar.gz - cd opensearch-dashboards{% endcomment %} + cd opensearch-dashboards ``` 1. If desired, modify `config/opensearch_dashboards.yml`. @@ -26,5 +27,3 @@ nav_order: 30 ```bash ./bin/opensearch-dashboards ``` - -1. See the [OpenSearch Dashboards documentation]({{site.url}}{{site.baseurl}}/dashboards/index/). From 0686ae05b57bd6409289cba469eae58f42e2343f Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Fri, 1 Oct 2021 01:21:16 -0700 Subject: [PATCH 045/229] Added upgrade tool docs --- _upgrade-to/upgrade-to.md | 94 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/_upgrade-to/upgrade-to.md b/_upgrade-to/upgrade-to.md index a415b621..e852fa1c 100644 --- a/_upgrade-to/upgrade-to.md +++ b/_upgrade-to/upgrade-to.md @@ -179,3 +179,97 @@ If you are upgrading an Open Distro for Elasticsearch cluster, we recommend firs } } ``` + +## Upgrade tool + +The `opensearch-upgrade` tool lets you automate some of the steps in [Upgrade to OpenSearch]({{site.url}}{{site.baseurl}}/upgrade-to/upgrade-to/#upgrade-to-opensearch), eliminating the need for error-prone manual operations. + +The `opensearch-upgrade` tool performs the following functions: + +- Imports any existing configurations and applies it to the new installation of OpenSearch. +- Installs any existing core plugins. + +### Limitations + +The `opensearch-upgrade` tool doesn't perform an end-to-end upgrade: + +- You need to run the tool on each node of the cluster individually as part of the upgrade process. +- The tool doesn't provide a rollback option after you've upgraded a node, so make sure you follow best practices and take backups. +- You must install all community plugins (if available) manually. +- The tool only validates any keystore settings at service start-up time, so you must manually remove any unsupported settings for the service to start. + +### Using the upgrade tool + +To perform a rolling upgrade using the [OpenSearch tarball]({{site.url}}{{site.baseurl}}/opensearch/install/tar/) distribution: + +Check [Upgrade paths]({{site.url}}{{site.baseurl}}/upgrade-to/upgrade-to/#upgrade-paths) to make sure that the version you’re upgrading to is supported and whether you need to upgrade to a supported Elasticsearch OSS version first. +{: .note } + +1. Disable shard allocation to prevent Elasticsearch OSS from replicating shards as you shut down nodes: + + ```json + PUT _cluster/settings + { + "persistent": { + "cluster.routing.allocation.enable": "primaries" + } + } + ``` + +1. On any one of the nodes, download and extract the OpenSearch tarball to a new directory. + +1. Make sure the following environment variables are set: + + - `ES_HOME` - Path to the existing Elasticsearch installation home. + - `ES_PATH_CONF` - Path to the existing Elasticsearch config directory. + - `OPENSEARCH_HOME` - Path to the OpenSearch installation home. + - `OPENSEARCH_PATH_CONF` - Path to the OpenSearch config directory. + +1. The `opensearch-upgrade` tool is in the `bin` directory of the distribution. Run the following command from the distribution home: + + Make sure you run this tool as the same user running the current Elasticsearch service. + {: .note } + + ```json + ./bin/opensearch-upgrade + ``` + +1. Stop Elasticsearch OSS on the node. + + On Linux distributions that use systemd, use this command: + + ```bash + sudo systemctl stop elasticsearch.service + ``` + + For tarball installations, find the process ID (`ps aux`) and kill it (`kill `). + +1. Start OpenSearch on the node: + + ```json + ./bin/opensearch -d. + ``` + +1. Repeat steps 2--6 until all nodes are using the new version. + +1. After all nodes are using the new version, re-enable shard allocation: + + ```json + PUT _cluster/settings + { + "persistent": { + "cluster.routing.allocation.enable": "all" + } + } + ``` + +### How it works + +Behind the scenes, the `opensearch-upgrade` tool performs the following tasks in sequence: + +1. Looks for a valid Elasticsearch installation on the current node. After it finds the installation, it reads the `elasticsearch.yml` file to get the endpoint details and connects to the locally running Elasticsearch service. If the tool can't find an Elasticsearch installation, it tries to get the path from the `ES_HOME` location. +1. Verifies if the existing version of Elasticsearch is compatible with the OpenSearch version. It prints a summary of the information gathered to the console and prompts you for a confirmation to proceed. +1. Imports the settings from the `elasticsearch.yml` config file into the `opensearch.yml` config file. +1. Copies across any custom JVM options from the `$ES_PATH_CONF/jvm.options.d` directory into the `$OPENSEARCH_PATH_CONF/jvm.options.d` directory . Similarly, it also imports the logging configurations from the `$ES_PATH_CONF/log4j2.properties` file into the `$OPENSEARCH_PATH_CONF/log4j2.properties` file. +1. Installs the core plugins that you’ve currently installed in the `$ES_HOME/plugins` directory. You must install all other third-party community plugins manually. +1. Imports the secure settings from the `elasticsearch.keystore` file (if any) into the `opensearch.keystore` file. If the keystore file is password protected, the `opensearch-upgrade` tool prompts you to enter the password. From 1f90f3ca523f57893ed30dac31db21cf5319ef4c Mon Sep 17 00:00:00 2001 From: aetter Date: Fri, 1 Oct 2021 09:46:47 -0700 Subject: [PATCH 046/229] Update Helm links --- _dashboards/install/helm.md | 2 +- _opensearch/install/helm.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/_dashboards/install/helm.md b/_dashboards/install/helm.md index 25936f1d..4d2e0c83 100644 --- a/_dashboards/install/helm.md +++ b/_dashboards/install/helm.md @@ -20,7 +20,7 @@ Resource | Description The specification in the default Helm chart supports many standard use cases and setups. You can modify the default chart to configure your desired specifications and set Transport Layer Security (TLS) and role-based access control (RBAC). For information about the default configuration, steps to configure security, and configurable parameters, see the -[README](https://github.com/opensearch-project/opensearch-devops/blob/main/Helm/README.md). +[README](https://github.com/opensearch-project/helm-charts/tree/main/charts). The instructions here assume you have a Kubernetes cluster with Helm preinstalled. See the [Kubernetes documentation](https://kubernetes.io/docs/setup/) for steps to configure a Kubernetes cluster and the [Helm documentation](https://helm.sh/docs/intro/install/) to install Helm. {: .note } diff --git a/_opensearch/install/helm.md b/_opensearch/install/helm.md index 55458626..33899cb0 100644 --- a/_opensearch/install/helm.md +++ b/_opensearch/install/helm.md @@ -20,7 +20,7 @@ Resource | Description The specification in the default Helm chart supports many standard use cases and setups. You can modify the default chart to configure your desired specifications and set Transport Layer Security (TLS) and role-based access control (RBAC). For information about the default configuration, steps to configure security, and configurable parameters, see the -[README](https://github.com/opensearch-project/opensearch-devops/blob/main/Helm/README.md). +[README](https://github.com/opensearch-project/helm-charts/tree/main/charts). The instructions here assume you have a Kubernetes cluster with Helm preinstalled. See the [Kubernetes documentation](https://kubernetes.io/docs/setup/) for steps to configure a Kubernetes cluster and the [Helm documentation](https://helm.sh/docs/intro/install/) to install Helm. {: .note } From 1fdf8b95179c8068b5c63fb4c5aa25d287d3b5e4 Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Fri, 1 Oct 2021 11:22:47 -0700 Subject: [PATCH 047/229] ad 1.1 --- _monitoring-plugins/ad/api.md | 126 ++++++++++++++++---------------- _monitoring-plugins/ad/index.md | 83 ++++++++++----------- 2 files changed, 101 insertions(+), 108 deletions(-) diff --git a/_monitoring-plugins/ad/api.md b/_monitoring-plugins/ad/api.md index a46420a9..e78c9723 100644 --- a/_monitoring-plugins/ad/api.md +++ b/_monitoring-plugins/ad/api.md @@ -240,56 +240,6 @@ POST _plugins/_anomaly_detection/detectors } ``` -To create a historical detector: - -#### Request - -```json -POST _plugins/_anomaly_detection/detectors -{ - "name": "test1", - "description": "test historical detector", - "time_field": "timestamp", - "indices": [ - "host-cloudwatch" - ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "feature_attributes": [ - { - "feature_name": "F1", - "feature_enabled": true, - "aggregation_query": { - "f_1": { - "sum": { - "field": "value" - } - } - } - } - ], - "detection_date_range": { - "start_time": 1577840401000, - "end_time": 1606121925000 - } -} -``` - You can specify the following options. Options | Description | Type | Required @@ -303,7 +253,6 @@ Options | Description | Type | Required `detection_interval` | The time interval for your anomaly detector. | `object` | Yes `window_delay` | Add extra processing time for data collection. | `object` | No `category_field` | Categorizes or slices data with a dimension. Similar to `GROUP BY` in SQL. | `list` | No -`detection_date_range` | Specify the start time and end time for a historical detector. | `object` | No --- @@ -316,10 +265,44 @@ Passes a date range to the anomaly detector to return any anomalies within that #### Request ```json -POST _plugins/_anomaly_detection/detectors//_preview +POST _plugins/_anomaly_detection/detectors/_preview + { - "period_start": 1588838250000, - "period_end": 1589443050000 + "period_start": 1612982516000, + "period_end": 1614278539000, + "detector": { + "name": "test-detector", + "description": "test nab_art_daily_jumpsdown", + "time_field": "timestamp", + "indices": [ + "nab_art_daily_jumpsdown" + ], + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "feature_attributes": [ + { + "feature_name": "F1", + "feature_enabled": true, + "aggregation_query": { + "f_1": { + "sum": { + "field": "value" + } + } + } + } + ] + } } ``` @@ -446,6 +429,17 @@ If you specify a category field, each result is associated with an entity: ``` +Or, you can specify the detector ID: + +```json +POST _plugins/_anomaly_detection/detectors/_preview +{ + "detector_id": "sYkUvHcBiZv51f-Lv8QN", + "period_start": 1612982516000, + "period_end": 1614278539000 +} +``` + --- ## Start detector job @@ -472,6 +466,15 @@ POST _plugins/_anomaly_detection/detectors//_start } ``` +To start historical analysis: + +```json +POST _plugins/_anomaly_detection/detectors//_start +{ + "start_time": 1503168590000, + "end_time": 1617301324000 +} +``` --- @@ -493,6 +496,12 @@ POST _plugins/_anomaly_detection/detectors//_stop Stopped detector: m4ccEnIBTXsGi3mvMt9p ``` +To stop historical analysis: + +```jsom +POST _plugins/_anomaly_detection/detectors//_stop?historical=true +``` + --- ## Search detector result @@ -786,15 +795,6 @@ POST _plugins/_anomaly_detection/detectors/results/_search } ``` -In historical detectors, specify the `detector_id`. -To get the latest task: - -#### Request - -```json -GET _plugins/_anomaly_detection/detectors/?task=true -``` - To query the anomaly results with `task_id`: #### Request diff --git a/_monitoring-plugins/ad/index.md b/_monitoring-plugins/ad/index.md index 5cb67b7a..cdd8fc89 100644 --- a/_monitoring-plugins/ad/index.md +++ b/_monitoring-plugins/ad/index.md @@ -17,24 +17,22 @@ Anomaly detection automatically detects anomalies in your OpenSearch data in ne You can pair the anomaly detection plugin with the [alerting plugin]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/) to notify you as soon as an anomaly is detected. -To use the anomaly detection plugin, your computer needs to have more than one CPU core. -{: .note } - ## Get started with Anomaly Detection To get started, choose **Anomaly Detection** in OpenSearch Dashboards. -To first test with sample streaming data, choose **Sample Detectors** and try out one of the preconfigured detectors. +To first test with sample streaming data, you can try out one of the preconfigured detectors with one of the sample datasets. -### Step 1: Create a detector +### Step 1: Define a detector -A detector is an individual anomaly detection task. You can create multiple detectors, and all the detectors can run simultaneously, with each analyzing data from different sources. +A detector is an individual anomaly detection task. You can define multiple detectors, and all the detectors can run simultaneously, with each analyzing data from different sources. 1. Choose **Create Detector**. 1. Enter a name and brief description. Make sure the name is unique and descriptive enough to help you to identify the purpose of the detector. 1. For **Data source**, choose the index you want to use as the data source. You can optionally use index patterns to choose multiple indices. +1. (Optional) For **Data filter**, filter the index you chose as the data source. From the **Data filter** menu, design your filter query by selecting **Field**, **Operator**, and **Value**, or choose **Use query DSL** and add your own JSON filter query. 1. Select the **Timestamp field** in your index. 1. (Optional) For **Data filter**, filter the index you chose as the data source. From the **Filter type** menu, choose **Visual filter**, and then design your filter query by selecting **Fields**, **Operator**, and **Value**, or choose **Custom Expression** and add your own JSON filter query. -1. For **Detector operation settings**, define the **Detector interval**, which is the time interval at which the detector collects data. +1. For **Operation settings**, define the **Detector interval**, which is the time interval at which the detector collects data. - The detector aggregates the data in this interval, then feeds the aggregated result into the anomaly detection model. The shorter you set this interval, the fewer data points the detector aggregates. The anomaly detection model uses a shingling process, a technique that uses consecutive data points to create a sample for the model. This process needs a certain number of aggregated data points from contiguous intervals. @@ -44,9 +42,9 @@ Set the window delay to shift the detector interval to account for this delay. - For example, say the detector interval is 10 minutes and data is ingested into your cluster with a general delay of 1 minute. Assume the detector runs at 2:00. The detector attempts to get the last 10 minutes of data from 1:50 to 2:00, but because of the 1-minute delay, it only gets 9 minutes of data and misses the data from 1:59 to 2:00. Setting the window delay to 1 minute shifts the interval window to 1:49 - 1:59, so the detector accounts for all 10 minutes of the detector interval time. -1. Choose **Create**. +1. Choose **Next**. -After you create the detector, the next step is to add features to it. +After you define the detector, the next step is to configure the model. ### Step 2: Add features to your detector @@ -54,24 +52,25 @@ A feature is the field in your index that you want to check for anomalies. A det For example, if you choose `min()`, the detector focuses on finding anomalies based on the minimum values of your feature. If you choose `average()`, the detector finds anomalies based on the average values of your feature. -A multi-feature model correlates anomalies across all its features. The [curse of dimensionality](https://en.wikipedia.org/wiki/Curse_of_dimensionality) makes it less likely for multi-feature models to identify smaller anomalies as compared to a single-feature model. Adding more features might negatively impact the [precision and recall](https://en.wikipedia.org/wiki/Precision_and_recall) of a model. A higher proportion of noise in your data might further amplify this negative impact. Selecting the optimal feature set is usually an iterative process. We recommend experimenting with a historical detector with different feature sets and checking the precision before moving on to real-time detectors. By default, the maximum number of features for a detector is 5. You can adjust this limit with the `plugins.anomaly_detection.max_anomaly_features` setting. +A multi-feature model correlates anomalies across all its features. The [curse of dimensionality](https://en.wikipedia.org/wiki/Curse_of_dimensionality) makes it less likely for multi-feature models to identify smaller anomalies as compared to a single-feature model. Adding more features might negatively impact the [precision and recall](https://en.wikipedia.org/wiki/Precision_and_recall) of a model. A higher proportion of noise in your data might further amplify this negative impact. Selecting the optimal feature set is usually an iterative process. By default, the maximum number of features for a detector is 5. You can adjust this limit with the `plugins.anomaly_detection.max_anomaly_features` setting. {: .note } -1. On the **Model configuration** page, enter the **Feature name**. -1. For **Find anomalies based on**, choose the method to find anomalies. For **Field Value** menu, choose the **field** and the **aggregation method**. Or choose **Custom expression**, and add your own JSON aggregation query. +1. On the **Configure Model** page, enter the **Feature name** and check **Enabled feature name**. +1. For **Find anomalies based on**, choose the method to find anomalies. For **Field Value**, choose the **aggregation method**. Or choose **Custom expression**, and add your own JSON aggregation query. +1. Select a field. -#### (Optional) Set a category field for high cardinality +#### (Optional) Set category fields for high cardinality You can categorize anomalies based on a keyword or IP field type. The category field categorizes or slices the source time series with a dimension like IP addresses, product IDs, country codes, and so on. This helps to see a granular view of anomalies within each entity of the category field to isolate and debug issues. -To set a category field, choose **Enable a category field** and select a field. +To set a category field, choose **Enable a category field** and select a field. You can’t change the category fields after you create the detector. Only a certain number of unique entities are supported in the category field. Use the following equation to calculate the recommended total number of entities supported in a cluster: ``` -(data nodes * heap size * anomaly detection maximum memory percentage) / (entity size of a detector) +(data nodes * heap size * anomaly detection maximum memory percentage) / (entity model size of a detector) ``` This formula provides a good starting point, but make sure to test with a representative workload. @@ -79,7 +78,7 @@ This formula provides a good starting point, but make sure to test with a repres For example, for a cluster with 3 data nodes, each with 8G of JVM heap size, a maximum memory percentage of 10% (default), and the entity size of the detector as 1MB: the total number of unique entities supported is (8.096 * 10^9 * 0.1 / 1M ) * 3 = 2429. -#### Set a shingle size +#### (Advanced settings) Set a shingle size Set the number of aggregation intervals from your data stream to consider in a detection window. It’s best to choose this value based on your actual data to see which one leads to the best results for your use case. @@ -92,10 +91,25 @@ For sample previews, the anomaly detection plugin selects a small number of data Examine the sample preview and use it to fine-tune your feature configurations (for example, enable or disable features) to get more accurate results. -1. Choose **Save and start detector**. -1. Choose between automatically starting the detector (recommended) or manually starting the detector at a later time. +1. Choose **Preview sample anomalies**. + - If you don't see any sample anomaly result, check the detector interval and make sure you have more than 400 data points for some entities during the preview date range. +1. Choose **Next**. -### Step 3: Observe the results +### Step 3: Set up detector jobs + +To start a real-time detector to find anomalies in your data in near real-time, check **Start real-time detector automatically (recommended)**. + +Alternatively, if you want to perform historical analysis and find patterns in long historical data windows (weeks or months), check **Run historical analysis detection** and select a date range (at least 128 detection intervals). + +Analyzing historical data helps you get familiar with the anomaly detection plugin. You can also evaluate the performance of a detector with historical data to further fine-tune it. + +We recommend experimenting with historical analysis with different feature sets and checking the precision before moving on to real-time detectors. + +### Step 4: Review and create + +Review your model configuration and select **Create detector**. + +### Step 5: Observe the results Choose the **Anomaly results** tab. You need to wait for some time to see the anomaly results. If the detector interval is 10 minutes, the detector might take more than an hour to start, as it's waiting for sufficient data to generate anomalies. @@ -106,7 +120,7 @@ If you see the detector pending in "initialization" for longer than a day, aggre ![Anomaly detection results]({{site.url}}{{site.baseurl}}/images/ad.png) -Analize anomalies with the following visualizations: +Analyze anomalies with the following visualizations: - **Live anomalies** - displays live anomaly results for the last 60 intervals. For example, if the interval is 10, it shows results for the last 600 minutes. The chart refreshes every 30 seconds. - **Anomaly history** - plots the anomaly grade with the corresponding measure of confidence. @@ -135,31 +149,10 @@ To see all the configuration settings for a detector, choose the **Detector conf 1. To make any changes to the detector configuration, or fine tune the time interval to minimize any false positives, go to the **Detector configuration** section and choose **Edit**. - You need to stop the detector to change its configuration. Confirm that you want to stop the detector and proceed. 1. To enable or disable features, in the **Features** section, choose **Edit** and adjust the feature settings as needed. After you make your changes, choose **Save and start detector**. -- Choose between automatically starting the detector (recommended) or manually starting the detector at a later time. -### Step 6: Analyze historical data +### Step 8: Manage your detectors -Analyzing historical data helps you get familiar with the anomaly detection plugin. You can also evaluate the performance of a detector with historical data to further fine-tune it. +To start, stop, or delete a detector, go to the **Detectors** page. -To use a historical detector, you need to specify a date range that has data present in at least 1,000 detection intervals. -{: .note } - -1. Choose **Historical detectors** and **Create historical detector**. -1. Enter the **Name** of the detector and a brief **Description**. -1. For **Data source**, choose the index to use as the data source. You can optionally use index patterns to choose multiple indices. -1. For **Time range**, select a time range for historical analysis. -1. For **Detector settings**, choose to use the settings of an existing detector. Or choose the **Timestamp field** in your index, add individual features to the detector, and set the detector interval. -1. (Optional) Choose to run the historical detector automatically after creating it. -1. Choose **Create**. - - You can stop the historical detector even before it completes. - -### Step 7: Manage your detectors - -To change or delete a detector, go to the **Detector details** page. - -1. To make changes to your detector, choose the detector name. -1. Choose **Actions** and **Edit detector**. - - You need to stop the detector to change its configuration. Confirm that you want to stop the detector and proceed. -1. Make your changes and choose **Save changes**. - -To delete your detector, choose **Actions** and **Delete detector**. In the pop-up box, type `delete` to confirm and choose **Delete**. +1. Choose the detector name. +2. Choose **Actions** and select **Start real-time detectors**, **Stop real-time detectors**, or **Delete detectors**. From dda5e2a53526df8869de69f42f8e71cf85f5399d Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Fri, 1 Oct 2021 11:26:24 -0700 Subject: [PATCH 048/229] fixed indendation feedback --- _upgrade-to/upgrade-to.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/_upgrade-to/upgrade-to.md b/_upgrade-to/upgrade-to.md index e852fa1c..0d554370 100644 --- a/_upgrade-to/upgrade-to.md +++ b/_upgrade-to/upgrade-to.md @@ -227,12 +227,12 @@ Check [Upgrade paths]({{site.url}}{{site.baseurl}}/upgrade-to/upgrade-to/#upgrad 1. The `opensearch-upgrade` tool is in the `bin` directory of the distribution. Run the following command from the distribution home: - Make sure you run this tool as the same user running the current Elasticsearch service. - {: .note } + Make sure you run this tool as the same user running the current Elasticsearch service. + {: .note } - ```json - ./bin/opensearch-upgrade - ``` + ```json + ./bin/opensearch-upgrade + ``` 1. Stop Elasticsearch OSS on the node. From cdff560cf6323894f560aae3a24f9caa0d8a8ad7 Mon Sep 17 00:00:00 2001 From: Liz Snyder Date: Fri, 1 Oct 2021 11:43:37 -0700 Subject: [PATCH 049/229] Incorporate feedback and random fixes --- _replication-plugin/api.md | 47 ++++++++++++++++-------------- _replication-plugin/auto-follow.md | 4 +-- _replication-plugin/get-started.md | 40 +++++++++++++++---------- _replication-plugin/index.md | 13 +++++---- _replication-plugin/permissions.md | 22 +++++++------- _replication-plugin/settings.md | 14 +++++---- 6 files changed, 79 insertions(+), 61 deletions(-) diff --git a/_replication-plugin/api.md b/_replication-plugin/api.md index a552d1c2..792aaecf 100644 --- a/_replication-plugin/api.md +++ b/_replication-plugin/api.md @@ -16,7 +16,7 @@ Use these replication operations to programmatically manage cross-cluster replic Introduced 1.1 {: .label .label-purple } -Initiate replication of an index from the leader cluster to the follower cluster. Run this operation on the follower cluster. +Initiate replication of an index from the leader cluster to the follower cluster. Send this request to the follower cluster. #### Request @@ -37,7 +37,7 @@ Specify the following options: Options | Description | Type | Required :--- | :--- |:--- |:--- | -`leader_alias` | The name of the leader cluster. | `string` | Yes +`leader_alias` | The name of the leader cluster. This alias is the same as the remote cluster name used to set up a cross-cluster connection. | `string` | Yes `leader_index` | The index on the leader cluster that you want to replicate. | `string` | Yes `use_roles` | The roles to use for all subsequent backend replication tasks between the indices. Specify a `leader_cluster_role` and `follower_cluster_role`. See [Map the leader and follower cluster roles]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). | `string` | If security plugin is enabled @@ -53,12 +53,13 @@ Options | Description | Type | Required Introduced 1.1 {: .label .label-purple } -Terminates replication and converts the follower index to a standard index. +Terminates replication and converts the follower index to a standard index. Send this request to the follower cluster. #### Request ```json POST /_plugins/_replication//_stop +{} ``` #### Sample response @@ -73,12 +74,13 @@ POST /_plugins/_replication//_stop Introduced 1.1 {: .label .label-purple } -Pauses replication of the leader index. If you don't resume replication after 12 hours, it stops completely and the follower index is converted to a standard index. +Pauses replication of the leader index. Send this request to the follower cluster. #### Request ```json -PUT /_plugins/_replication//_pause +POST /_plugins/_replication//_pause +{} ``` #### Sample response @@ -93,12 +95,13 @@ PUT /_plugins/_replication//_pause Introduced 1.1 {: .label .label-purple } -Resumes replication of the leader index. +Resumes replication of the leader index. Send this request to the follower cluster. #### Request ```json -PUT /_plugins/_replication//_resume +POST /_plugins/_replication//_resume +{} ``` #### Sample response @@ -113,7 +116,7 @@ PUT /_plugins/_replication//_resume Introduced 1.1 {: .label .label-purple } -Gets the status of index replication. You can use this API to measure replication lag. Run this command from the leader cluster. +Gets the status of index replication. Possible statuses are `SYNCING`, `BOOTSTRAPING`, `PAUSED`, and `REPLICATION NOT IN PROGRESS`. Use the syncing details to measure replication lag. Send this request to the follower cluster. #### Request @@ -125,20 +128,20 @@ GET /_plugins/_replication//_status ```json { - "status":"SYNCING", - "reason":"User initiated", - "remote_cluster":"remote-cluster", - "leader_index":"leader-01", - "follower_index":"follower-01", - "syncing_details":{ - "remote_checkpoint": 19, - "local_checkpoint": 19, - "seq_no": 20 - } + "status" : "SYNCING", + "reason" : "User initiated", + "leader_alias" : "leader-cluster", + "leader_index" : "leader-01", + "follower_index" : "follower-01", + "syncing_details" : { + "leader_checkpoint" : 19, + "follower_checkpoint" : 19, + "seq_no" : 0 + } } ``` -To include shard replication details in the response, add `&verbose=true`. +To include shard replication details in the response, add the `&verbose=true` parameter. ## Update settings Introduced 1.1 @@ -170,9 +173,9 @@ PUT /_plugins/_replication//_update Introduced 1.1 {: .label .label-purple } -Automatically starts replication on indices matching a specified pattern. Newly created indices on the remote cluster that match one of the specified patterns will be automatically configured as follower indices. You can also use this API to update existing auto-follow patterns. +Automatically starts replication on indices matching a specified pattern. If a new index on the leader cluster matches the pattern, OpenSearch automatically creates a follower index and begins replication. You can also use this API to update existing replication rules. -Run this command on the follower cluster. +Send this request to the follower cluster. Make sure to note the names of all auto-follow patterns after you create them. The replication plugin currently does not include an API operation to retrieve a list of existing patterns. {: .tip } @@ -215,7 +218,7 @@ Introduced 1.1 Deletes the specified replication rule. This operation prevents any new indices from being replicated but does not stop existing replication that the rule has already initiated. -Run this command on the follower cluster. +Send this request to the follower cluster. #### Request diff --git a/_replication-plugin/auto-follow.md b/_replication-plugin/auto-follow.md index 2af9551c..2e06761e 100644 --- a/_replication-plugin/auto-follow.md +++ b/_replication-plugin/auto-follow.md @@ -42,7 +42,7 @@ curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://loc }' ``` -If the security plugin is disabled, you can leave out the `use_roles` parameter. If it's enabled, however, you need to specify the leader and follower cluster roles that OpenSearch will use to authenticate the request. This example uses `all_access` for simplicity, but we recommend creating a replication user on each cluster and [mapping it accordingly]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). +If the security plugin is disabled, you can leave out the `use_roles` parameter. If it's enabled, however, you need to specify the leader and follower cluster roles that OpenSearch uses to authenticate requests. This example uses `all_access` for simplicity, but we recommend creating a replication user on each cluster and [mapping it accordingly]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). {: .tip } To test the rule, create a matching index on the leader cluster: @@ -64,7 +64,7 @@ yellow open movies-0001 kHOxYYHxRMeszLjTD9rvSQ 1 1 0 ## Delete a replication rule -When you delete a replication rule, OpenSearch stops replicating *new* indices that match the pattern, but replication of existing indices that the rule previously created will continue. If you need to stop existing replication activity, use the [stop replication API operation]({{site.url}}{{site.baseurl}}/replication-plugin/api/#stop-replication). +When you delete a replication rule, OpenSearch stops replicating *new* indices that match the pattern, but existing indices that the rule previously created continue to replicate. If you need to stop existing replication activity, use the [stop replication API operation]({{site.url}}{{site.baseurl}}/replication-plugin/api/#stop-replication). ```bash curl -XDELETE -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/_autofollow?pretty' -d ' diff --git a/_replication-plugin/get-started.md b/_replication-plugin/get-started.md index 34e2899a..30955b8c 100644 --- a/_replication-plugin/get-started.md +++ b/_replication-plugin/get-started.md @@ -6,13 +6,13 @@ nav_order: 10 # Get started with cross-cluster replication -With cross-cluster replication, you index data to a leader index and that data is replicated to one or more read-only follower indices. All subsequent operations on the leader are replicated on the follower, such as creating, updating, or deleting documents. +With cross-cluster replication, you index data to a leader index, and OpenSearch replicates that data to one or more read-only follower indices. All subsequent operations on the leader are replicated on the follower, such as creating, updating, or deleting documents. ## Prerequisites Cross-cluster replication has the following prerequisites: -- Install the replication plugin on all nodes of both the leader and the follower cluster. -- If you've overridden `node.roles` in opensearch.yml on the remote cluster, make sure it also includes the `remote_cluster_client` role: +- Both the leader and follower cluster must have the replication plugin installed. +- If you've overridden `node.roles` in `opensearch.yml` on the remote cluster, make sure it also includes the `remote_cluster_client` role: ```yaml node.roles: [, remote_cluster_client] @@ -20,11 +20,11 @@ Cross-cluster replication has the following prerequisites: ## Permissions -Make sure the security plugin is either enabled on both clusters or disabled on both clusters. If you disabled the security plugin, you can skip this section. +Make sure the security plugin is either enabled on both clusters or disabled on both clusters. If you disabled the security plugin, you can skip this section. However, we strongly recommend enabling the security plugin in production scenarios. If the security plugin is enabled, non-admin users need to be mapped to the appropriate permissions in order to perform replication actions. For index and cluster-level permissions requirements, see [Cross-cluster replication permissions]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/). -In addition, add the following setting to opensearch.yml on the leader cluster so it allows connections from the follower cluster: +In addition, add the following setting to `opensearch.yml` on the leader cluster so it allows connections from the follower cluster: ```yml plugins.security.nodes_dn_dynamic_config_enabled: true @@ -32,6 +32,9 @@ plugins.security.nodes_dn_dynamic_config_enabled: true ## Example setup +The following example demonstrates how to replicate data between two single-node clusters: `leader-cluster` on port 9201, and `follower-cluster` on port 9200. +{% comment %} + Save this sample file as `docker-compose.yml` and run `docker-compose up` to start two single-node clusters on the same network: ```yml @@ -86,6 +89,8 @@ networks: After the clusters start, verify the names of each: +{% endcomment %} + ```bash curl -XGET -u 'admin:admin' -k 'https://localhost:9201' { @@ -102,6 +107,8 @@ curl -XGET -u 'admin:admin' -k 'https://localhost:9200' } ``` +{% comment %} + For this example, use port 9201 (`replication-node1`) as the leader and port 9200 (`replication-node2`) as the follower cluster. To get the IP address for the leader cluster, first identify its container ID: @@ -119,10 +126,13 @@ Then get that container's IP address: docker inspect --format='{% raw %}{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}{% endraw %}' 731f5e8b0f4b 172.22.0.3 ``` +{% endcomment %} ## Set up a cross-cluster connection -On the follower cluster, add the leader cluster name and the IP address (with port 9300) for each seed node. In this case, you only have one seed node: +Cross-cluster replication follows a "pull" model, so most changes occur on the follower cluster, not the leader cluster. + +On the follower cluster, add the leader cluster name and the IP address (with port 9300) for each seed node. Because this is a single-node cluster, you only have one seed node: ```bash curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_cluster/settings?pretty' -d ' @@ -141,15 +151,13 @@ curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://loca ## Start replication -To get started, create an index called `leader-01` on the remote (leader) cluster: +To get started, create an index called `leader-01` on the leader cluster: ```bash curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/leader-01?pretty' ``` -Start replication of that index from the follower cluster. Starting replication creates a follower index from scratch; you can't convert an existing index to a follower index. - -Provide the leader cluster and index that you want to replicate: +Then start replication of that index from the follower cluster. In the request body, provide the leader cluster and index, along with the security roles that you want to use: ```bash curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_start?pretty' -d ' @@ -163,10 +171,12 @@ curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://loca }' ``` -If the security plugin is disabled, you can leave out the `use_roles` parameter. If it's enabled, however, you need to specify the leader and follower cluster roles that OpenSearch will use to authenticate the request. This example uses `all_access` for simplicity, but we recommend creating a replication user on each cluster and [mapping it accordingly]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). +If the security plugin is disabled, omit the `use_roles` parameter. If it's enabled, however, you must specify the leader and follower cluster roles that OpenSearch will use to authenticate the request. This example uses `all_access` for simplicity, but we recommend creating a replication user on each cluster and [mapping it accordingly]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). {: .tip } -This command creates an identical read-only index named "follower-01" on the local cluster that continuously stays updated with changes to the "leader-01" index on the remote cluster. +This command creates an identical read-only index named `follower-01` on the local cluster that continuously stays updated with changes to the `leader-01` index on the remote cluster. Starting replication creates a follower index from scratch; you can't convert an existing index to a follower index. + +## Confirm replication After replication starts, get the status: @@ -187,7 +197,7 @@ curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/fol } ``` -## Confirm replication +Possible statuses are `SYNCING`, `BOOTSTRAPING`, `PAUSED`, and `REPLICATION NOT IN PROGRESS`. The leader and follower checkpoint values increment with each change and illustrate how many updates the follower is behind the leader. If the indices are fully synced, the values are the same. To confirm that replication is actually happening, add a document to the leader index: @@ -244,8 +254,6 @@ curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://loc When replication resumes, the follower index picks up any changes that were made to the leader index while replication was paused. -If you don't resume replication within 12 hours, replication stops completely and the follower index is converted to a standard index. - ## Stop replication Terminate replication of a specified index from the follower cluster: @@ -254,7 +262,7 @@ Terminate replication of a specified index from the follower cluster: curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_stop' -d '{}' ``` -When you stop replication, the follower index un-follows the leader and becomes a standard index that you can write to. You can't restart replication after it's been terminated. +When you stop replication, the follower index un-follows the leader and becomes a standard index that you can write to. You can't restart replication after stopping it. Get the status to confirm that the index is no longer being replicated: diff --git a/_replication-plugin/index.md b/_replication-plugin/index.md index 07ba256f..f3a99d08 100644 --- a/_replication-plugin/index.md +++ b/_replication-plugin/index.md @@ -8,12 +8,15 @@ has_children: false # Cross-cluster replication -The cross-cluster replication plugin lets you replicate indices, mappings, and metadata from one OpenSearch cluster to another. It follows an active-passive replication model where the follower index (where the data is replicated) pulls data from the leader (source) index. +The cross-cluster replication plugin lets you replicate indices, mappings, and metadata from one OpenSearch cluster to another. Cross-cluster replication has the following benefits: +- By replicating your indices, you ensure that you can continue to handle search requests in the event of an outage. +- Replicating data across geographically distant data centers minimizes the distance between the data and the application server, reducing expensive latencies. +- You can replicate data from multiple smaller clusters to a centralized reporting cluster, which is useful when it's inefficient to query across a large network. -The replication plugin supports replication of indices using wildcard pattern matching and provides commands to pause, resume, and stop replication. Once replication starts on an index, it initiates a persistent background task on the primary shard on the follower cluster that continuously polls corresponding shards from the leader cluster for updates. +Replication follows an active-passive model where the follower index (where the data is replicated) pulls data from the leader (remote) index. -The replication plugin integrates with the security plugin so you can encrypt cross-cluster traffic with node-to-node encryption and control access to replication activities. - -To start, see [Get started with cross-cluster replication]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/). +The replication plugin supports replication of indices using wildcard pattern matching and provides commands to pause, resume, and stop replication. Once replication starts on an index, it initiates persistent background tasks on all primary shards on the follower cluster, which continuously poll corresponding shards from the leader cluster for updates. +You can use the replication plugin with the security plugin to encrypt cross-cluster traffic with node-to-node encryption and control access to replication activities. +To start, see [Get started with cross-cluster replication]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/). \ No newline at end of file diff --git a/_replication-plugin/permissions.md b/_replication-plugin/permissions.md index 6b3804ad..2204dba5 100644 --- a/_replication-plugin/permissions.md +++ b/_replication-plugin/permissions.md @@ -14,24 +14,26 @@ Because cross-cluster replication involves multiple clusters, it's possible that - Security plugin enabled only for TLS on both clusters (`plugins.security.ssl_only`) - Security plugin absent or disabled on both clusters (not recommended) -You can enable node-to-node encryption on both the leader and the follower cluster to ensure that replication traffic between the clusters is encrypted. +Enable node-to-node encryption on both the leader and the follower cluster to ensure that replication traffic between the clusters is encrypted. ## Basic permissions -In order for non-admin users to perform replication activities, they need to be mapped to the appropriate permissions. +In order for non-admin users to perform replication activities, they be mapped to the appropriate permissions. -The security plugin has two built-in roles that cover most replication use cases: `cross_cluster_replication_leader_full_access` which provides replication permissions on the leader cluster, and `cross_cluster_replication_follower_full_access` which provides replication permissions on the follower cluster. For descriptions of each, see [Predefined roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/users-roles#predefined-roles). +The security plugin has two built-in roles that cover most replication use cases: `cross_cluster_replication_leader_full_access`, which provides replication permissions on the leader cluster, and `cross_cluster_replication_follower_full_access`, which provides replication permissions on the follower cluster. For descriptions of each, see [Predefined roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/users-roles#predefined-roles). If you don't want to use the default roles, you can combine individual replication [permissions]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#replication-permissions) to meet your needs. Most permissions correspond to specific REST API operations. For example, the `indices:admin/plugins/replication/index/pause` permission lets you pause replication. ## Map the leader and follower cluster roles -The [start replication]({{site.url}}{{site.baseurl}}/replication-plugin/api/#start-replication) and [create replication rule]({{site.url}}{{site.baseurl}}/replication-plugin/api/#start-replication) are special cases because they involve background processes that require permissions on both the leader and the follower cluster. Therefore, when you perform one of these actions, you need to explicitly pass the `leader_cluster_role` and -`follower_cluster_role` in the request, which OpenSearch will then use in all backend replication tasks. +associates roles passed in the request to these replication jobs to run in the background -To enable non-admins to start replication and create replication rules, create an identical user on each cluster (for example, `replication_user`) and map them to the `cross_cluster_replication_leader_full_access` role on the remote cluster and `cross_cluster_replication_follower_full_access` on the follower cluster. For instructions, see [Map users to roles]({{site.url}}{{site.baseurl}}/access-control/users-roles/#map-users-to-roles). +The [start replication]({{site.url}}{{site.baseurl}}/replication-plugin/api/#start-replication) and [create replication rule]({{site.url}}{{site.baseurl}}/replication-plugin/api/#create-replication-rule) operations are special cases. They involve background processes on the leader and follower clusters that must be associated with roles. When you perform one of these actions, you must explicitly pass the `leader_cluster_role` and +`follower_cluster_role` in the request, which OpenSearch then uses in all backend replication tasks. -You can then pass those roles into the request along with the appropriate credentials: +To enable non-admins to start replication and create replication rules, create an identical user on each cluster (for example, `replication_user`) and map them to the `cross_cluster_replication_leader_full_access` role on the remote cluster and `cross_cluster_replication_follower_full_access` on the follower cluster. For instructions, see [Map users to roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/users-roles/#map-users-to-roles). + +Then add those roles to the request, and sign it with the appropriate credentials: ```bash curl -XPUT -k -H 'Content-Type: application/json' -u 'replication_user:password' 'https://localhost:9200/_plugins/_replication/follower-01/_start?pretty' -d ' @@ -45,7 +47,7 @@ curl -XPUT -k -H 'Content-Type: application/json' -u 'replication_user:password' }' ``` -You can instead choose to create your own leader and follower cluster roles to meet your needs, but we recommend using the default roles. +You can create your own, custom leader and follower cluster roles using individual permissions, but we recommend using the default roles, which are a good fit for most use cases. ## Replication permissions @@ -53,7 +55,7 @@ The following sections list the available index and cluster-level permissions fo ### Follower cluster -Users can have the following permissions for the follower cluster: +The security plugin supports these permissions for the follower cluster: ``` indices:admin/plugins/replication/index/setup/validate @@ -69,7 +71,7 @@ cluster:admin/plugins/replication/autofollow/update ### Leader cluster -Users can have the following permissions for the leader cluster: +The security plugin supports these permissions for the leader cluster: ``` indices:admin/plugins/replication/validate diff --git a/_replication-plugin/settings.md b/_replication-plugin/settings.md index 66d16ec4..b4815b44 100644 --- a/_replication-plugin/settings.md +++ b/_replication-plugin/settings.md @@ -10,13 +10,13 @@ The replication plugin adds several settings to the standard OpenSearch cluster The settings are dynamic, so you can change the default behavior of the plugin without restarting your cluster. You can mark settings as `persistent` or `transient`. -For example, to update the retention period of the result index: +For example, to update how often the follower cluster polls the leader cluster for updates: ```json PUT _cluster/settings { "persistent": { - "plugins.replication.indices.recovery.parallel_chunks": "8" + "plugins.replication.follower.metadata_sync_interval": "30s" } } ``` @@ -25,8 +25,10 @@ These settings manage the resources consumed by remote recoveries. We don’t re Setting | Default | Description :--- | :--- | :--- -`plugins.replication.indices.recovery.chunk_size` | 1MB | The chunk size requested by the follower cluster during file transfer. Specify the chunk size as a value and unit, for example, 10MB, 5KB. -`plugins.replication.indices.recovery.parallel_chunks` | 5 | The number of file chunk requests that can be sent in parallel for each recovery. -`plugins.replication.indices.recovery.request_timeout` | 60s | The amount of time to wait for individual network requests during the remote recovery process. A single action timeout can cause recovery to fail. -`plugins.replication.indices.recovery.activity_timeout` | 5m | The amount of time to wait for recovery activity. If the leader cluster doesn't receive recovery requests from the follower for this amount of time, it closes the in-memory resources needed to supply data to the follower during recovery. +`plugins.replication.follower.index.recovery.chunk_size` | 10MB | The chunk size requested by the follower cluster during file transfer. Specify the chunk size as a value and unit, for example, 10MB, 5KB. See [Supported units]({{site.url}}{{site.baseurl}}/opensearch/units/). +`plugins.replication.follower.index.recovery.max_concurrent_file_chunks` | 4 | The number of file chunk requests that can be sent in parallel for each recovery. +`plugins.replication.follower.index.ops_batch_size` | 5000 | The number of operations that can be fetched at a time during the syncing phase of replication. +`plugins.replication.follower.concurrent_readers_per_shard` | 2 | The number of concurrent requests from the follower cluster per shard during the syncing phase of replication. +`plugins.replication.autofollow.fetch_poll_interval` | 30s | How often auto-follow tasks poll the leader cluster for new matching indices. +`plugins.replication.follower.metadata_sync_interval` | 60s | How often the follower cluster polls the leader cluster for updated index metadata. From bfadd04a182e24ebbbefde670703987f5a9b998f Mon Sep 17 00:00:00 2001 From: keithhc2 Date: Fri, 1 Oct 2021 16:41:31 -0700 Subject: [PATCH 050/229] Added a small section for the visual editor --- _im-plugin/ism/index.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/_im-plugin/ism/index.md b/_im-plugin/ism/index.md index f27e2636..0104dc68 100644 --- a/_im-plugin/ism/index.md +++ b/_im-plugin/ism/index.md @@ -31,14 +31,21 @@ To get started, choose **Index Management** in OpenSearch Dashboards. A policy is a set of rules that describes how an index should be managed. For information about creating a policy, see [Policies]({{site.url}}{{site.baseurl}}/im-plugin/ism/policies/). +You can use the JSON editor or visual editor to create policies. Compared to the JSON editor, the visual editor offers a more structured way of defining policies by separating the process into creating error notifications, defining ISM templates, and adding states. We recommend using the visual editor if you want to see pre-defined fields such as which actions you can assign to a state or under what conditions a state can transition into a destination state. + +#### JSON editor + 1. Choose the **Index Policies** tab. 2. Choose **Create policy**. -3. In the **Name policy** section, enter a policy ID. -4. In the **Define policy** section, enter your policy. -5. Choose **Create**. +3. Choose **JSON editor**. +4. In the **Name policy** section, enter a policy ID. +5. In the **Define policy** section, enter your policy. +6. Choose **Create**. -After you create a policy, your next step is to attach this policy to an index or indices. -You can set up an `ism_template` in the policy so when you create an index that matches the ISM template pattern, the index will have this policy attached to it: +After you create a policy, your next step is to attach it to an index or indices. +You can set up an `ism_template` in the policy so when an index that matches the ISM template pattern is created, the plugin automatically attaches the policy to the index. + +The following example demonstrates how to create a policy that automatically gets attached to all indices whose names start with `index_name-`. ```json PUT _plugins/_ism/policies/policy_id From 57d7ee0aaa40002b9f0a99878530f220144ada30 Mon Sep 17 00:00:00 2001 From: Omurbek Date: Sat, 2 Oct 2021 23:53:48 +0300 Subject: [PATCH 051/229] Added missed "template" settings --- _im-plugin/ism/policies.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/_im-plugin/ism/policies.md b/_im-plugin/ism/policies.md index ec4bf737..afad9396 100644 --- a/_im-plugin/ism/policies.md +++ b/_im-plugin/ism/policies.md @@ -558,9 +558,11 @@ The following sample template policy is for a rollover use case. PUT _index_template/ism_rollover { "index_patterns": ["log*"], - "settings": { - "plugins.index_state_management.rollover_alias": "log" - } + "template": { + "settings": { + "opendistro.index_state_management.rollover_alias": "log" + } + } } ``` @@ -586,6 +588,12 @@ The following sample template policy is for a rollover use case. } ``` +5. Verify if the policy is attached to the `log-000001` index: + + ```json + GET _opendistro/_ism/explain/log-000001?pretty + ``` + ## Example policy The following example policy implements a `hot`, `warm`, and `delete` workflow. You can use this policy as a template to prioritize resources to your indices based on their levels of activity. From 9f74fed5828cd50c945995fc442eb5b593269a91 Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Sun, 3 Oct 2021 10:35:18 -0700 Subject: [PATCH 052/229] more updates --- _monitoring-plugins/ad/api.md | 2233 +++++++++++++++++++++++++--- _monitoring-plugins/ad/index.md | 19 +- _monitoring-plugins/ad/settings.md | 17 +- 3 files changed, 2049 insertions(+), 220 deletions(-) diff --git a/_monitoring-plugins/ad/api.md b/_monitoring-plugins/ad/api.md index e78c9723..2a8427e3 100644 --- a/_monitoring-plugins/ad/api.md +++ b/_monitoring-plugins/ad/api.md @@ -272,10 +272,10 @@ POST _plugins/_anomaly_detection/detectors/_preview "period_end": 1614278539000, "detector": { "name": "test-detector", - "description": "test nab_art_daily_jumpsdown", + "description": "test server_log", "time_field": "timestamp", "indices": [ - "nab_art_daily_jumpsdown" + "server_log" ], "detection_interval": { "period": { @@ -795,7 +795,7 @@ POST _plugins/_anomaly_detection/detectors/results/_search } ``` -To query the anomaly results with `task_id`: +You can query the anomaly results of a historical detector with the `task_id`: #### Request @@ -885,7 +885,7 @@ GET _plugins/_anomaly_detection/detectors/results/_search "time_field": "timestamp", "last_update_time": 1612126640448, "indices": [ - "nab_art_daily_jumpsdown" + "server_log" ], "window_delay": { "period": { @@ -939,6 +939,192 @@ GET _plugins/_anomaly_detection/detectors/results/_search } ``` +You can specify the following options. + +Options | Description | Type | Required +:--- | :--- |:--- |:--- | +`anomalyThreshold` | Filter out low anomaly grade results. Default is -1. The lowest possible anomaly grade is 0. -1 means that the detector returns all results. | `float` | No +`dateRangeFilter` | Specify the date range in:
- `startTime` (int): Start time to collect results. Recorded in milliseconds since the Unix Epoch.
- `endTime` (int): End time to collect results. Recorded in milliseconds since the Unix Epoch.
- `fieldName` (string): The field that you want to match the start and end time. | `object` | Yes +`entity` | If not empty, the parameter contains the entity name and value. Default is empty.
- `name` (string): Field name that you want to search in.
- `value` (string): Entity value that you want to search for. | `object` | No +`sort` | If not empty, sorts the result by a field in a certain order. Default is empty. Properties of `sort`:
- `direction` (string): Specify "desc" or "asc" for descending or ascending order.
- `field` (string): Order the results by a field. | `object` | No + +--- + +## Search detector tasks +Introduced 1.1 +{: .label .label-purple } + +Searches detector tasks. + +#### Request + +```json +POST _plugins/_anomaly_detection/detectors/tasks/_search +{ + "query": { + "bool": { + "filter": [ + { + "term": { + "detector_id": { + "value": "_6WPu3cBBnauGn7oxUAv" + } + } + }, + { + "term": { + "task_type": { + "value": "HISTORICAL_HC_DETECTOR" + } + } + } + ] + } + }, + "sort": [ + { + "execution_start_time": { + "order": "desc" + } + } + ] +} +``` + + +#### Sample response + +```json +{ + "took" : 5, + "timed_out" : false, + "_shards" : { + "total" : 1, + "successful" : 1, + "skipped" : 0, + "failed" : 0 + }, + "hits" : { + "total" : { + "value" : 1, + "relation" : "eq" + }, + "max_score" : null, + "hits" : [ + { + "_index" : ".opendistro-anomaly-detection-state", + "_type" : "_doc", + "_id" : "TM3tOHwBCi2h__AOXlyQ", + "_version" : 3, + "_seq_no" : 14, + "_primary_term" : 1, + "_score" : null, + "_source" : { + "detector_id" : "rlDtOHwBD5tpxlbyW7Nt", + "error" : "", + "detection_date_range" : { + "start_time" : 1632437852100, + "end_time" : 1633042652100 + }, + "task_progress" : 0.5, + "last_update_time" : 1633042667358, + "execution_start_time" : 1633042652810, + "state" : "RUNNING", + "coordinating_node" : "2hEGbUw6ShaiKe05n_xLdA", + "task_type" : "HISTORICAL_HC_DETECTOR", + "started_by" : "admin", + "init_progress" : 0.0, + "is_latest" : true, + "detector" : { + "category_field" : [ + "type" + ], + "description" : "test", + "ui_metadata" : { + "features" : { + "test-feature" : { + "aggregationBy" : "sum", + "aggregationOf" : "value", + "featureType" : "simple_aggs" + } + }, + "filters" : [ ] + }, + "feature_attributes" : [ + { + "feature_id" : "7VDtOHwBD5tpxlbyWqPs", + "feature_enabled" : true, + "feature_name" : "test-feature", + "aggregation_query" : { + "test_feature" : { + "sum" : { + "field" : "value" + } + } + } + } + ], + "schema_version" : 0, + "time_field" : "timestamp", + "last_update_time" : 1633042652012, + "indices" : [ + "server_log" + ], + "window_delay" : { + "period" : { + "unit" : "Minutes", + "interval" : 1 + } + }, + "detection_interval" : { + "period" : { + "unit" : "Minutes", + "interval" : 5 + } + }, + "name" : "test-detector", + "filter_query" : { + "match_all" : { + "boost" : 1.0 + } + }, + "shingle_size" : 8, + "user" : { + "backend_roles" : [ + "admin" + ], + "custom_attribute_names" : [ ], + "roles" : [ + "own_index", + "all_access" + ], + "name" : "admin", + "user_requested_tenant" : null + }, + "detector_type" : "MULTI_ENTITY" + }, + "user" : { + "backend_roles" : [ + "admin" + ], + "custom_attribute_names" : [ ], + "roles" : [ + "own_index", + "all_access" + ], + "name" : "admin", + "user_requested_tenant" : "__user__" + } + }, + "sort" : [ + 1633042652810 + ] + } + ] + } +} +``` + --- @@ -976,6 +1162,218 @@ DELETE _plugins/_anomaly_detection/detectors/ } ``` +--- + +## Delete detector results +Introduced 1.1 +{: .label .label-purple } + +Deletes a detector results based on a query. + +#### Request + +```json +DELETE _plugins/_anomaly_detection/detectors/results + +{ + "query": { + "bool": { + "filter": [ + { + "term": { + "detector_id": { + "value": "rlDtOHwBD5tpxlbyW7Nt" + } + } + }, + { + "term": { + "task_id": { + "value": "TM3tOHwBCi2h__AOXlyQ" + } + } + }, + { + "range": { + "data_start_time": { + "lte": 1632441600000 + } + } + } + ] + } + } +} +``` + + +#### Sample response + +```json +{ + "took" : 48, + "timed_out" : false, + "total" : 28, + "updated" : 0, + "created" : 0, + "deleted" : 28, + "batches" : 1, + "version_conflicts" : 0, + "noops" : 0, + "retries" : { + "bulk" : 0, + "search" : 0 + }, + "throttled_millis" : 0, + "requests_per_second" : -1.0, + "throttled_until_millis" : 0, + "failures" : [ ] +} +``` + +--- + +## Validate detector +Introduced 1.1 +{: .label .label-purple } + +Validates detector before creating. This API shows you any invalid fields in your configuration and also recommendations on how to fix it. + +#### Request + +```json +POST _plugins/_anomaly_detection/detectors/_validate/detector,model +{ + "name": "test-detector", + "description": "Test detector", + "time_field": "timestamp", + "indices": [ + "order*" + ], + "feature_attributes": [ + { + "feature_name": "total_order", + "feature_enabled": true, + "aggregation_query": { + "total_order": { + "sum": { + "field": "value" + } + } + } + } + ], + "filter_query": { + "bool": { + "filter": [ + { + "exists": { + "field": "value", + "boost": 1 + } + } + ], + "adjust_pure_negative": true, + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "category_field": [ + "hc_field" + ], + "shingle_size": 8 +} +``` + + +#### Sample response + +```json +{ + "detector": { + "name": { + "message": "name should be set|duplicate" + }, + "time_field": { + "message": "time_field should be set missing|not_exist" + }, + "indices": { + "message": "should be set|not_exist|empty" + }, + "feature_attributes": { + // exist when message is "there exists non-numeric field" + [Optional]"problematic_feature_name1": { + "message": "{field} is invalid with {exceptionMessage}" + } + "problematic_feature_name2": { + "message": "{field} is invalid with {exceptionMessage}" + }, + "message": "there exists non-numeric field|duplicate feature names|over 5 features|duplicate feature aggregation query names" + }, + "detection_interval": { + "message": "detection_interval should be set|Interval should be non-negative|unit is not supported" + }, + "category_field": { + "message": "must only 1 field, and must be IP address or keyword type" + }, + "shingle_size": { + "message": "must be between 1 and 1000" + }, + }, + "model": { + "filter_query": { + "message": "data is too sparse after filter_query is applied" + }, + "detection_interval": { + // exists when suggested value can be found + [Optional]"suggested_value": { + "period": { + "interval": 1, + "unit": "Minutes" + } + } + "message": "use suggested value|no suggested value found, ingest more data" + }, + "category_field": { + "message": "data with {category_field} is too sparse, ingest more data" + }, + "feature_attributes": { + "problematic_feature_name1": { + "message": "data is too sparse, ingest more data with this {field}" + } + "problematic_feature_name2": { + "message": "data is too sparse, ingest more data with this {field}" + }, + "message": "data is too sparse, ingest more data" + }, + "memory": { + "message": "model size exceeds memory limit, please stop/delete unused detectors, or reduce shingle size or number of features" + }, + "window_delay": { + // exists when suggested value can be found + [Optional]"suggested_value": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "message": "use suggested value(if it exists), and ingest more data if possible" + } + } +} + +``` --- @@ -1100,56 +1498,6 @@ PUT _plugins/_anomaly_detection/detectors/ } ``` -To update a historical detector: - -#### Request - -```json -PUT _plugins/_anomaly_detection/detectors/ -{ - "name": "test1", - "description": "test historical detector", - "time_field": "timestamp", - "indices": [ - "nab_art_daily_jumpsdown" - ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "feature_attributes": [ - { - "feature_name": "F1", - "feature_enabled": true, - "aggregation_query": { - "f_1": { - "sum": { - "field": "value" - } - } - } - } - ], - "detection_date_range": { - "start_time": 1577840401000, - "end_time": 1606121925000 - } -} -``` - --- ## Get detector @@ -1238,80 +1586,133 @@ GET _plugins/_anomaly_detection/detectors/?job=true ```json { - "_id" : "m4ccEnIBTXsGi3mvMt9p", - "_version" : 1, - "_primary_term" : 1, - "_seq_no" : 3, - "anomaly_detector" : { - "name" : "test-detector", - "description" : "Test detector", - "time_field" : "timestamp", - "indices" : [ - "order*" - ], - "filter_query" : { - "bool" : { - "filter" : [ - { - "exists" : { - "field" : "value", - "boost" : 1.0 - } - } + "_id": "LJxGsXcBoDQA8W1Q--A1", + "_version": 1, + "_primary_term": 1, + "_seq_no": 0, + "anomaly_detector": { + "name": "test2", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" ], - "adjust_pure_negative" : true, - "boost" : 1.0 - } - }, - "detection_interval" : { - "period" : { - "interval" : 1, - "unit" : "Minutes" - } - }, - "window_delay" : { - "period" : { - "interval" : 1, - "unit" : "Minutes" - } - }, - "schema_version" : 0, - "feature_attributes" : [ - { - "feature_id" : "mYccEnIBTXsGi3mvMd8_", - "feature_name" : "total_order", - "feature_enabled" : true, - "aggregation_query" : { - "total_order" : { - "sum" : { - "field" : "value" + "filter_query": { + "match_all": { + "boost": 1.0 } - } + }, + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "K5xGsXcBoDQA8W1Q-uCF", + "feature_name": "F1", + "feature_enabled": *true*, + "aggregation_query": { + "f_1": { + "sum": { + "field": "value" + } + } + } + } + ], + "last_update_time": 1613586955060, + "detector_type": "MULTI_ENTITY" + }, + "anomaly_detector_job": { + "name": "LJxGsXcBoDQA8W1Q--A1", + "schedule": { + "interval": { + "start_time": 1613587220387, + "period": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "enabled": *false*, + "enabled_time": 1613587220387, + "last_update_time": 1613587289169, + "lock_duration_seconds": 60, + "disabled_time": 1613587289169 + }, + "anomaly_detection_task": { + "task_id": "WZ5LsXcBoDQA8W1QmUa3", + "last_update_time": 1613587349022, + "error": "Task cancelled by user", + "state": "STOPPED", + "detector_id": "LJxGsXcBoDQA8W1Q--A1", + "task_progress": 0.26321793, + "init_progress": 1.0, + "current_piece": 1611030900000, + "execution_start_time": 1613587257783, + "execution_end_time": 1613587349022, + "is_latest": *true*, + "task_type": "HISTORICAL", + "coordinating_node": "NSw5j-3YQeGkH8KESVKlzw", + "worker_node": "NSw5j-3YQeGkH8KESVKlzw", + "detector": { + "name": "test2", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" + ], + "filter_query": { + "match_all": { + "boost": 1.0 + } + }, + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "K5xGsXcBoDQA8W1Q-uCF", + "feature_name": "F1", + "feature_enabled": *true*, + "aggregation_query": { + "f_1": { + "sum": { + "field": "value" + } + } + } + } + ], + "last_update_time": 1613586955060, + "detector_type": "MULTI_ENTITY" } - } - ], - "last_update_time" : 1589441737319 - }, - "anomaly_detector_job" : { - "name" : "m4ccEnIBTXsGi3mvMt9p", - "schedule" : { - "interval" : { - "start_time" : 1589442051271, - "period" : 1, - "unit" : "Minutes" - } - }, - "window_delay" : { - "period" : { - "interval" : 1, - "unit" : "Minutes" - } - }, - "enabled" : true, - "enabled_time" : 1589442051271, - "last_update_time" : 1589442051271, - "lock_duration_seconds" : 60 - } + } } ``` @@ -1336,7 +1737,7 @@ GET _plugins/_anomaly_detection/detectors/?task=true "description": "test", "time_field": "timestamp", "indices": [ - "nab*" + "ser*" ], "filter_query": { "match_all": { @@ -1419,7 +1820,7 @@ GET _plugins/_anomaly_detection/detectors/?task=true "description": "test", "time_field": "timestamp", "indices": [ - "nab*" + "ser*" ], "filter_query": { "match_all": { @@ -1513,16 +1914,24 @@ GET _plugins/_anomaly_detection/detectors/_search POST _plugins/_anomaly_detection/detectors/_search Sample Input: + { "query": { - "match": { - "name": "test-detector" + "bool": { + "filter": [ + { + "terms": { + "indices": [ + "server_log" + ] + } + } + ] } } } ``` - #### Sample response ```json @@ -1627,82 +2036,864 @@ GET _plugins/_anomaly_detection/stats/ ```json { - "_nodes" : { - "total" : 3, - "successful" : 3, - "failed" : 0 - }, - "cluster_name" : "multi-node-run", - "anomaly_detectors_index_status" : "green", - "detector_count" : 1, - "models_checkpoint_index_status" : "green", - "anomaly_results_index_status" : "green", - "nodes" : { - "IgWDUfzFRzW0FWAXM5FGJw" : { - "ad_execute_request_count" : 8, - "ad_execute_failure_count" : 7, - "models" : [ + "anomaly_detectors_index_status": "yellow", + "anomaly_detection_state_status": "yellow", + "single_entity_detector_count": 0, + "detector_count": 1, + "multi_entity_detector_count": 0, + "anomaly_detection_job_index_status": "yellow", + "models_checkpoint_index_status": "yellow", + "anomaly_results_index_status": "yellow", + "nodes": { + "hhfW2ZNVTJCtbs8rO-nF4g": { + "ad_execute_request_count": 6, + "models": [ { - "detector_id" : "m4ccEnIBTXsGi3mvMt9p", - "model_type" : "rcf", - "model_id" : "m4ccEnIBTXsGi3mvMt9p_model_rcf_0" + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578975, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_ic43SHH11NWr32xXgjRFwg", + "last_checkpoint_time": 1629827339211, + "entity": [ + { + "name": "host", + "value": "server_2" + }, + { + "name": "service", + "value": "app_6" + } + ] }, { - "detector_id" : "m4ccEnIBTXsGi3mvMt9p", - "model_type" : "threshold", - "model_id" : "m4ccEnIBTXsGi3mvMt9p_model_threshold" - } - ] - }, - "y7YUQWukQEWOYbfdEq13hQ" : { - "ad_execute_request_count" : 0, - "ad_execute_failure_count" : 0, - "models" : [ ] - }, - "cDcGNsPoRAyRMlPP1m-vZw" : { - "ad_execute_request_count" : 0, - "ad_execute_failure_count" : 0, - "models" : [ - { - "detector_id" : "m4ccEnIBTXsGi3mvMt9p", - "model_type" : "rcf", - "model_id" : "m4ccEnIBTXsGi3mvMt9p_model_rcf_2" + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578975, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_k2gq2eRP0vTV2LNNyFdIqg", + "last_checkpoint_time": 1629827339733, + "entity": [ + { + "name": "host", + "value": "server_1" + }, + { + "name": "service", + "value": "app_0" + } + ] }, { - "detector_id" : "m4ccEnIBTXsGi3mvMt9p", - "model_type" : "rcf", - "model_id" : "m4ccEnIBTXsGi3mvMt9p_model_rcf_1" + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578980, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_DOze7d0HnK3K54g3Emk1XA", + "last_checkpoint_time": 1629827343186, + "entity": [ + { + "name": "host", + "value": "server_2" + }, + { + "name": "service", + "value": "app_3" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578977, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_NgGmglQvOMQQciDdPxN_Ig", + "last_checkpoint_time": 1629827340961, + "entity": [ + { + "name": "host", + "value": "server_3" + }, + { + "name": "service", + "value": "app_5" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578977, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_oTdcenY1L5bqa6chUxg7xw", + "last_checkpoint_time": 1629827340263, + "entity": [ + { + "name": "host", + "value": "server_1" + }, + { + "name": "service", + "value": "app_1" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578979, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_zxSqAWv5Iz19v-Hnqhrwrw", + "last_checkpoint_time": 1629827342814, + "entity": [ + { + "name": "host", + "value": "server_1" + }, + { + "name": "service", + "value": "app_5" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578976, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_Yu8X2rA39lhjYzqebjLxhQ", + "last_checkpoint_time": 1629827339992, + "entity": [ + { + "name": "host", + "value": "server_2" + }, + { + "name": "service", + "value": "app_1" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578978, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_6SvF11RCqf7HYbY56BnFKA", + "last_checkpoint_time": 1629827341806, + "entity": [ + { + "name": "host", + "value": "server_3" + }, + { + "name": "service", + "value": "app_2" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578980, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_3S8k6q_DLFhw3hboko3dfw", + "last_checkpoint_time": 1629827343371, + "entity": [ + { + "name": "host", + "value": "server_1" + }, + { + "name": "service", + "value": "app_3" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578978, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_0uafBokvEYuncGbjP3D2qA", + "last_checkpoint_time": 1629827342302, + "entity": [ + { + "name": "host", + "value": "server_2" + }, + { + "name": "service", + "value": "app_5" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578977, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_93DEK2PooWlHF6gkh-0hIA", + "last_checkpoint_time": 1629827340727, + "entity": [ + { + "name": "host", + "value": "server_2" + }, + { + "name": "service", + "value": "app_4" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578975, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_FuqXh0HBXlPhKepOc6JADQ", + "last_checkpoint_time": 1629827338908, + "entity": [ + { + "name": "host", + "value": "server_3" + }, + { + "name": "service", + "value": "app_6" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578979, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_B4zrbSQ1-pvdBLx0FzQxvw", + "last_checkpoint_time": 1629827342611, + "entity": [ + { + "name": "host", + "value": "server_3" + }, + { + "name": "service", + "value": "app_3" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578978, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_xm_gKBMKlgymKcoqZyXT8A", + "last_checkpoint_time": 1629827341365, + "entity": [ + { + "name": "host", + "value": "server_2" + }, + { + "name": "service", + "value": "app_0" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578978, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_OnZ4CP-yJF5llO57gUjM6w", + "last_checkpoint_time": 1629827341599, + "entity": [ + { + "name": "host", + "value": "server_3" + }, + { + "name": "service", + "value": "app_1" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578977, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_E_uWreoeJpGrAMMaitg8BA", + "last_checkpoint_time": 1629827340418, + "entity": [ + { + "name": "host", + "value": "server_3" + }, + { + "name": "service", + "value": "app_4" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578979, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_OIsxy2nXMVdngK6Vv3X0uw", + "last_checkpoint_time": 1629827342444, + "entity": [ + { + "name": "host", + "value": "server_2" + }, + { + "name": "service", + "value": "app_2" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578978, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_I0L8K8ktyVnyL59CVFCLVQ", + "last_checkpoint_time": 1629827342068, + "entity": [ + { + "name": "host", + "value": "server_1" + }, + { + "name": "service", + "value": "app_4" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578975, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_ZoNYVJsq5ry6e-SWXmAt1Q", + "last_checkpoint_time": 1629827339435, + "entity": [ + { + "name": "host", + "value": "server_1" + }, + { + "name": "service", + "value": "app_6" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578978, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_qo2ANH_NS7Bg8iV4AJpHOw", + "last_checkpoint_time": 1629827341187, + "entity": [ + { + "name": "host", + "value": "server_3" + }, + { + "name": "service", + "value": "app_0" + } + ] + }, + { + "detector_id": "mmZFeXsB7JcKN0mdnMf4", + "model_type": "entity", + "last_used_time": 1629827578980, + "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_412FoQwCykWTAhjVfDGQDg", + "last_checkpoint_time": 1629827342983, + "entity": [ + { + "name": "host", + "value": "server_1" + }, + { + "name": "service", + "value": "app_2" + } + ] } - ] + ], + "ad_canceled_batch_task_count": 0, + "ad_hc_execute_request_count": 6, + "ad_hc_execute_failure_count": 0, + "model_count": 21, + "ad_execute_failure_count": 0, + "ad_batch_task_failure_count": 0, + "ad_total_batch_task_execution_count": 0, + "ad_executing_batch_task_count": 0 } } } ``` +The `model_count` parameter shows the total number of models running on each node’s memory. Historical detectors contain additional fields: +- `ad_total_batch_task_execution_count` +- `ad_executing_batch_task_count` +- `ad_canceled_batch_task_count` +- `ad_batch_task_failure_count` + #### Sample response ```json { - "anomaly_detectors_index_status": "yellow", - "anomaly_detection_state_status": "yellow", - "historical_detector_count": 3, - "detector_count": 7, - "anomaly_detection_job_index_status": "yellow", - "models_checkpoint_index_status": "yellow", - "anomaly_results_index_status": "yellow", + "anomaly_detectors_index_status": "green", + "anomaly_detection_state_status": "green", + "single_entity_detector_count": 0, + "detector_count": 1, + "multi_entity_detector_count": 1, + "anomaly_detection_job_index_status": "green", + "models_checkpoint_index_status": "green", + "anomaly_results_index_status": "green", "nodes": { - "Mz9HDZnuQwSCw0UiisxwWg": { + "bCtWtxWPThq0BIn5P5I4Xw": { "ad_execute_request_count": 0, - "models": [], - "ad_canceled_batch_task_count": 2, + "models": [ + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152729, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error20", + "last_checkpoint_time": 1633043556222, + "entity": [ + { + "name": "type", + "value": "error20" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152767, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error31", + "last_checkpoint_time": 1633043855146, + "entity": [ + { + "name": "type", + "value": "error31" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152729, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error21", + "last_checkpoint_time": 1633043555143, + "entity": [ + { + "name": "type", + "value": "error21" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152727, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error13", + "last_checkpoint_time": 1633043554046, + "entity": [ + { + "name": "type", + "value": "error13" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152753, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error24", + "last_checkpoint_time": 1633043853986, + "entity": [ + { + "name": "type", + "value": "error24" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152792, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error25", + "last_checkpoint_time": 1633043857320, + "entity": [ + { + "name": "type", + "value": "error25" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152779, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error28", + "last_checkpoint_time": 1633043856244, + "entity": [ + { + "name": "type", + "value": "error28" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152732, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error16", + "last_checkpoint_time": 1633043557253, + "entity": [ + { + "name": "type", + "value": "error16" + } + ] + } + ], + "ad_canceled_batch_task_count": 0, "ad_hc_execute_request_count": 0, "ad_hc_execute_failure_count": 0, + "model_count": 8, "ad_execute_failure_count": 0, "ad_batch_task_failure_count": 0, - "ad_executing_batch_task_count": 1, - "ad_total_batch_task_count": 8 + "ad_total_batch_task_execution_count": 15, + "ad_executing_batch_task_count": 3 + }, + "dIyavWhmSYWGz65b4u-lpQ": { + "ad_execute_request_count": 0, + "models": [ + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152729, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error3", + "last_checkpoint_time": 1633043256013, + "entity": [ + { + "name": "type", + "value": "error3" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152727, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error1", + "last_checkpoint_time": 1633043254819, + "entity": [ + { + "name": "type", + "value": "error1" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152735, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error22", + "last_checkpoint_time": 1633043557023, + "entity": [ + { + "name": "type", + "value": "error22" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152750, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error32", + "last_checkpoint_time": 1633043854080, + "entity": [ + { + "name": "type", + "value": "error32" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152784, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error23", + "last_checkpoint_time": 1633043857463, + "entity": [ + { + "name": "type", + "value": "error23" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152774, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error26", + "last_checkpoint_time": 1633043856308, + "entity": [ + { + "name": "type", + "value": "error26" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152734, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error14", + "last_checkpoint_time": 1633043555939, + "entity": [ + { + "name": "type", + "value": "error14" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152731, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error9", + "last_checkpoint_time": 1633043257214, + "entity": [ + { + "name": "type", + "value": "error9" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152730, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error19", + "last_checkpoint_time": 1633043553882, + "entity": [ + { + "name": "type", + "value": "error19" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152732, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error18", + "last_checkpoint_time": 1633043554874, + "entity": [ + { + "name": "type", + "value": "error18" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152763, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error29", + "last_checkpoint_time": 1633043855226, + "entity": [ + { + "name": "type", + "value": "error29" + } + ] + } + ], + "ad_canceled_batch_task_count": 0, + "ad_hc_execute_request_count": 0, + "ad_hc_execute_failure_count": 0, + "model_count": 11, + "ad_execute_failure_count": 0, + "ad_batch_task_failure_count": 0, + "ad_total_batch_task_execution_count": 14, + "ad_executing_batch_task_count": 3 + }, + "2hEGbUw6ShaiKe05n_xLdA": { + "ad_execute_request_count": 5, + "models": [ + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152714, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error5", + "last_checkpoint_time": 1633043256689, + "entity": [ + { + "name": "type", + "value": "error5" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152711, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error6", + "last_checkpoint_time": 1633043254281, + "entity": [ + { + "name": "type", + "value": "error6" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152716, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error4", + "last_checkpoint_time": 1633043257797, + "entity": [ + { + "name": "type", + "value": "error4" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152709, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error2", + "last_checkpoint_time": 1633043260938, + "entity": [ + { + "name": "type", + "value": "error2" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152742, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error30", + "last_checkpoint_time": 1633043853983, + "entity": [ + { + "name": "type", + "value": "error30" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152725, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error11", + "last_checkpoint_time": 1633043263038, + "entity": [ + { + "name": "type", + "value": "error11" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152712, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error10", + "last_checkpoint_time": 1633043255533, + "entity": [ + { + "name": "type", + "value": "error10" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152719, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error7", + "last_checkpoint_time": 1633043258826, + "entity": [ + { + "name": "type", + "value": "error7" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152708, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error8", + "last_checkpoint_time": 1633043259841, + "entity": [ + { + "name": "type", + "value": "error8" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152721, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error12", + "last_checkpoint_time": 1633043261989, + "entity": [ + { + "name": "type", + "value": "error12" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152720, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error15", + "last_checkpoint_time": 1633043553786, + "entity": [ + { + "name": "type", + "value": "error15" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152724, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error17", + "last_checkpoint_time": 1633043554909, + "entity": [ + { + "name": "type", + "value": "error17" + } + ] + }, + { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "model_type": "entity", + "last_used_time": 1633044152751, + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error27", + "last_checkpoint_time": 1633043855105, + "entity": [ + { + "name": "type", + "value": "error27" + } + ] + } + ], + "ad_canceled_batch_task_count": 0, + "ad_hc_execute_request_count": 5, + "ad_hc_execute_failure_count": 0, + "model_count": 13, + "ad_execute_failure_count": 0, + "ad_batch_task_failure_count": 0, + "ad_total_batch_task_execution_count": 14, + "ad_executing_batch_task_count": 3 } } } @@ -1946,7 +3137,7 @@ It also helps track the initialization percentage, the required shingles, and th GET _plugins/_anomaly_detection/detectors//_profile/ GET _plugins/_anomaly_detection/detectors//_profile?_all=true GET _plugins/_anomaly_detection/detectors//_profile/ -GET /_plugins/_anomaly_detection/detectors//_profile/, +GET _plugins/_anomaly_detection/detectors//_profile/, ``` #### Sample Responses @@ -1963,35 +3154,648 @@ GET _plugins/_anomaly_detection/detectors//_profile?_all=true&pretty { "state": "RUNNING", + "error": "", "models": [ { - "model_id": "cneh7HEBHPICjJIdXdrR_model_rcf_2", - "model_size_in_bytes": 4456448, - "node_id": "VS29z70PSzOdHiEw4SoV9Q" + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error20", + "entity": [ + { + "name": "type", + "value": "error20" + } + ], + "model_size_in_bytes": 403491, + "node_id": "bCtWtxWPThq0BIn5P5I4Xw" }, { - "model_id": "cneh7HEBHPICjJIdXdrR_model_rcf_1", - "model_size_in_bytes": 4456448, - "node_id": "VS29z70PSzOdHiEw4SoV9Q" + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error31", + "entity": [ + { + "name": "type", + "value": "error31" + } + ], + "model_size_in_bytes": 403491, + "node_id": "bCtWtxWPThq0BIn5P5I4Xw" }, { - "model_id": "cneh7HEBHPICjJIdXdrR_model_threshold", - "node_id": "Og23iUroTdKrkwS-y89zLw" + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error21", + "entity": [ + { + "name": "type", + "value": "error21" + } + ], + "model_size_in_bytes": 403491, + "node_id": "bCtWtxWPThq0BIn5P5I4Xw" }, { - "model_id": "cneh7HEBHPICjJIdXdrR_model_rcf_0", - "model_size_in_bytes": 4456448, - "node_id": "Og23iUroTdKrkwS-y89zLw" + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error13", + "entity": [ + { + "name": "type", + "value": "error13" + } + ], + "model_size_in_bytes": 403491, + "node_id": "bCtWtxWPThq0BIn5P5I4Xw" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error24", + "entity": [ + { + "name": "type", + "value": "error24" + } + ], + "model_size_in_bytes": 403491, + "node_id": "bCtWtxWPThq0BIn5P5I4Xw" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error25", + "entity": [ + { + "name": "type", + "value": "error25" + } + ], + "model_size_in_bytes": 403491, + "node_id": "bCtWtxWPThq0BIn5P5I4Xw" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error28", + "entity": [ + { + "name": "type", + "value": "error28" + } + ], + "model_size_in_bytes": 403491, + "node_id": "bCtWtxWPThq0BIn5P5I4Xw" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error16", + "entity": [ + { + "name": "type", + "value": "error16" + } + ], + "model_size_in_bytes": 403491, + "node_id": "bCtWtxWPThq0BIn5P5I4Xw" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error3", + "entity": [ + { + "name": "type", + "value": "error3" + } + ], + "model_size_in_bytes": 403491, + "node_id": "dIyavWhmSYWGz65b4u-lpQ" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error1", + "entity": [ + { + "name": "type", + "value": "error1" + } + ], + "model_size_in_bytes": 403491, + "node_id": "dIyavWhmSYWGz65b4u-lpQ" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error22", + "entity": [ + { + "name": "type", + "value": "error22" + } + ], + "model_size_in_bytes": 403491, + "node_id": "dIyavWhmSYWGz65b4u-lpQ" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error32", + "entity": [ + { + "name": "type", + "value": "error32" + } + ], + "model_size_in_bytes": 403491, + "node_id": "dIyavWhmSYWGz65b4u-lpQ" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error23", + "entity": [ + { + "name": "type", + "value": "error23" + } + ], + "model_size_in_bytes": 403491, + "node_id": "dIyavWhmSYWGz65b4u-lpQ" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error26", + "entity": [ + { + "name": "type", + "value": "error26" + } + ], + "model_size_in_bytes": 403491, + "node_id": "dIyavWhmSYWGz65b4u-lpQ" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error14", + "entity": [ + { + "name": "type", + "value": "error14" + } + ], + "model_size_in_bytes": 403491, + "node_id": "dIyavWhmSYWGz65b4u-lpQ" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error9", + "entity": [ + { + "name": "type", + "value": "error9" + } + ], + "model_size_in_bytes": 403491, + "node_id": "dIyavWhmSYWGz65b4u-lpQ" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error19", + "entity": [ + { + "name": "type", + "value": "error19" + } + ], + "model_size_in_bytes": 403491, + "node_id": "dIyavWhmSYWGz65b4u-lpQ" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error18", + "entity": [ + { + "name": "type", + "value": "error18" + } + ], + "model_size_in_bytes": 403491, + "node_id": "dIyavWhmSYWGz65b4u-lpQ" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error29", + "entity": [ + { + "name": "type", + "value": "error29" + } + ], + "model_size_in_bytes": 403491, + "node_id": "dIyavWhmSYWGz65b4u-lpQ" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error5", + "entity": [ + { + "name": "type", + "value": "error5" + } + ], + "model_size_in_bytes": 403491, + "node_id": "2hEGbUw6ShaiKe05n_xLdA" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error6", + "entity": [ + { + "name": "type", + "value": "error6" + } + ], + "model_size_in_bytes": 403491, + "node_id": "2hEGbUw6ShaiKe05n_xLdA" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error4", + "entity": [ + { + "name": "type", + "value": "error4" + } + ], + "model_size_in_bytes": 403491, + "node_id": "2hEGbUw6ShaiKe05n_xLdA" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error2", + "entity": [ + { + "name": "type", + "value": "error2" + } + ], + "model_size_in_bytes": 403491, + "node_id": "2hEGbUw6ShaiKe05n_xLdA" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error30", + "entity": [ + { + "name": "type", + "value": "error30" + } + ], + "model_size_in_bytes": 403491, + "node_id": "2hEGbUw6ShaiKe05n_xLdA" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error11", + "entity": [ + { + "name": "type", + "value": "error11" + } + ], + "model_size_in_bytes": 403491, + "node_id": "2hEGbUw6ShaiKe05n_xLdA" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error10", + "entity": [ + { + "name": "type", + "value": "error10" + } + ], + "model_size_in_bytes": 403491, + "node_id": "2hEGbUw6ShaiKe05n_xLdA" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error7", + "entity": [ + { + "name": "type", + "value": "error7" + } + ], + "model_size_in_bytes": 403491, + "node_id": "2hEGbUw6ShaiKe05n_xLdA" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error8", + "entity": [ + { + "name": "type", + "value": "error8" + } + ], + "model_size_in_bytes": 403491, + "node_id": "2hEGbUw6ShaiKe05n_xLdA" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error12", + "entity": [ + { + "name": "type", + "value": "error12" + } + ], + "model_size_in_bytes": 403491, + "node_id": "2hEGbUw6ShaiKe05n_xLdA" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error15", + "entity": [ + { + "name": "type", + "value": "error15" + } + ], + "model_size_in_bytes": 403491, + "node_id": "2hEGbUw6ShaiKe05n_xLdA" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error17", + "entity": [ + { + "name": "type", + "value": "error17" + } + ], + "model_size_in_bytes": 403491, + "node_id": "2hEGbUw6ShaiKe05n_xLdA" + }, + { + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error27", + "entity": [ + { + "name": "type", + "value": "error27" + } + ], + "model_size_in_bytes": 403491, + "node_id": "2hEGbUw6ShaiKe05n_xLdA" } ], - "shingle_size": 8, - "coordinating_node": "Og23iUroTdKrkwS-y89zLw", - "total_size_in_bytes": 13369344, + "total_size_in_bytes": 12911712, "init_progress": { - "percentage": "70%", - "estimated_minutes_left": 77, - "needed_shingles": 77 - } + "percentage": "100%" + }, + "total_entities": 33, + "active_entities": 32, + "ad_task": { + "ad_task": { + "task_id": "Os4HOXwBCi2h__AONgpc", + "last_update_time": 1633044347855, + "started_by": "admin", + "state": "RUNNING", + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "task_progress": 0, + "init_progress": 0, + "execution_start_time": 1633044346460, + "is_latest": true, + "task_type": "HISTORICAL_HC_DETECTOR", + "coordinating_node": "2hEGbUw6ShaiKe05n_xLdA", + "detector": { + "name": "test-detector", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" + ], + "filter_query": { + "match_all": { + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 5, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "7VDtOHwBD5tpxlbyWqPs", + "feature_name": "test-feature", + "feature_enabled": true, + "aggregation_query": { + "test_feature": { + "sum": { + "field": "value" + } + } + } + } + ], + "ui_metadata": { + "features": { + "test-feature": { + "aggregationBy": "sum", + "aggregationOf": "value", + "featureType": "simple_aggs" + } + }, + "filters": [] + }, + "last_update_time": 1633042652012, + "category_field": [ + "type" + ], + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": null + }, + "detector_type": "MULTI_ENTITY" + }, + "detection_date_range": { + "start_time": 1632437820000, + "end_time": 1633042620000 + }, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + } + }, + "node_id": "2hEGbUw6ShaiKe05n_xLdA", + "task_id": "Os4HOXwBCi2h__AONgpc", + "task_type": "HISTORICAL_HC_DETECTOR", + "detector_task_slots": 10, + "total_entities_count": 32, + "pending_entities_count": 22, + "running_entities_count": 10, + "running_entities": [ + "error9", + "error8", + "error7", + "error6", + "error5", + "error4", + "error32", + "error31", + "error30", + "error3" + ], + "entity_task_profiles": [ + { + "shingle_size": 8, + "rcf_total_updates": 994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "bCtWtxWPThq0BIn5P5I4Xw", + "entity": [ + { + "name": "type", + "value": "error6" + } + ], + "task_id": "P84HOXwBCi2h__AOOgrC", + "task_type": "HISTORICAL_HC_ENTITY" + }, + { + "shingle_size": 8, + "rcf_total_updates": 994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "bCtWtxWPThq0BIn5P5I4Xw", + "entity": [ + { + "name": "type", + "value": "error5" + } + ], + "task_id": "QM4HOXwBCi2h__AOOgre", + "task_type": "HISTORICAL_HC_ENTITY" + }, + { + "shingle_size": 8, + "rcf_total_updates": 994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "bCtWtxWPThq0BIn5P5I4Xw", + "entity": [ + { + "name": "type", + "value": "error9" + } + ], + "task_id": "PM4HOXwBCi2h__AOOgp3", + "task_type": "HISTORICAL_HC_ENTITY" + }, + { + "shingle_size": 8, + "rcf_total_updates": 994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "dIyavWhmSYWGz65b4u-lpQ", + "entity": [ + { + "name": "type", + "value": "error31" + } + ], + "task_id": "LM4HOXwBCi2h__AOOw7v", + "task_type": "HISTORICAL_HC_ENTITY" + }, + { + "shingle_size": 8, + "rcf_total_updates": 994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "dIyavWhmSYWGz65b4u-lpQ", + "entity": [ + { + "name": "type", + "value": "error4" + } + ], + "task_id": "Kc4HOXwBCi2h__AOOw6Y", + "task_type": "HISTORICAL_HC_ENTITY" + }, + { + "shingle_size": 8, + "rcf_total_updates": 994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "dIyavWhmSYWGz65b4u-lpQ", + "entity": [ + { + "name": "type", + "value": "error30" + } + ], + "task_id": "Lc4HOXwBCi2h__AOPA4R", + "task_type": "HISTORICAL_HC_ENTITY" + }, + { + "shingle_size": 8, + "rcf_total_updates": 994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "dIyavWhmSYWGz65b4u-lpQ", + "entity": [ + { + "name": "type", + "value": "error8" + } + ], + "task_id": "Pc4HOXwBCi2h__AOOgqJ", + "task_type": "HISTORICAL_HC_ENTITY" + }, + { + "shingle_size": 8, + "rcf_total_updates": 994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "2hEGbUw6ShaiKe05n_xLdA", + "entity": [ + { + "name": "type", + "value": "error3" + } + ], + "task_id": "Fs4HOXwBCi2h__AOPBLH", + "task_type": "HISTORICAL_HC_ENTITY" + }, + { + "shingle_size": 8, + "rcf_total_updates": 994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "2hEGbUw6ShaiKe05n_xLdA", + "entity": [ + { + "name": "type", + "value": "error32" + } + ], + "task_id": "Ks4HOXwBCi2h__AOOw7D", + "task_type": "HISTORICAL_HC_ENTITY" + }, + { + "shingle_size": 8, + "rcf_total_updates": 994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "2hEGbUw6ShaiKe05n_xLdA", + "entity": [ + { + "name": "type", + "value": "error7" + } + ], + "task_id": "Ps4HOXwBCi2h__AOOgqh", + "task_type": "HISTORICAL_HC_ENTITY" + } + ] + }, + "model_count": 32 } GET _plugins/_anomaly_detection/detectors//_profile/total_size_in_bytes @@ -2001,6 +3805,8 @@ GET _plugins/_anomaly_detection/detectors//_profile/total_size_in_by } ``` +The `model_count` parameter shows the total number of models that a detector runs in memory. This is useful if you have several models running on your cluster and want to know the count. + If you configured the category field, you can see the number of unique values in the field and all active entities with models running in memory. You can use this data to estimate how much memory is required for anomaly detection so you can decide how to size your cluster. For example, if a detector has one million entities and only 10 of them are active in memory, you need to scale your cluster up or out. @@ -2008,7 +3814,7 @@ You can use this data to estimate how much memory is required for anomaly detect #### Request ```json -GET /_plugins/_anomaly_detection/detectors//_profile?_all=true&pretty +GET _plugins/_anomaly_detection/detectors//_profile?_all=true&pretty { "state": "RUNNING", @@ -2058,6 +3864,10 @@ GET /_plugins/_anomaly_detection/detectors//_profile?_all=true&prett } ``` +The `total_entities` parameter shows you the total number of entities including the number of category fields for a detector. + +Getting the total count of entities is an expensive operation for a detector with more than one category field. By default, a real-time detector counts the number of entities up to a value of 10,000 and a historical detector counts the number of entities up to a value of 1,000. + The `profile` operation also provides information about each entity, such as the entity’s `last_sample_timestamp` and `last_active_timestamp`. If there are no anomaly results for an entity, either the entity doesn't have any sample data or its model is removed from the model cache. @@ -2067,7 +3877,20 @@ If there are no anomaly results for an entity, either the entity doesn't have an #### Request ```json -GET /_plugins/_anomaly_detection/detectors//_profile?_all=true&entity=i-00f28ec1eb8997686 +GET _plugins/_anomaly_detection/detectors//_profile?_all=true +{ + "entity": [ + { + "name": "host", + "value": "i-00f28ec1eb8997686" + } + ] +} +``` + +#### Sample Responses + +```json { "category_field": "host", "value": "i-00f28ec1eb8997686", @@ -2116,7 +3939,7 @@ GET _plugins/_anomaly_detection/detectors//_profile/ad_task "description": "test", "time_field": "timestamp", "indices": [ - "nab_art_daily_jumpsdown" + "server_log" ], "filter_query": { "match_all": { diff --git a/_monitoring-plugins/ad/index.md b/_monitoring-plugins/ad/index.md index cdd8fc89..87417e0f 100644 --- a/_monitoring-plugins/ad/index.md +++ b/_monitoring-plugins/ad/index.md @@ -73,10 +73,14 @@ Only a certain number of unique entities are supported in the category field. Us (data nodes * heap size * anomaly detection maximum memory percentage) / (entity model size of a detector) ``` +To get the entity model size of a detector, use the [profile detector API]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/api/#profile-detector). You can adjust the maximum memory percentage with the `plugins.anomaly_detection.model_max_size_percent` setting. + This formula provides a good starting point, but make sure to test with a representative workload. {: .note } -For example, for a cluster with 3 data nodes, each with 8G of JVM heap size, a maximum memory percentage of 10% (default), and the entity size of the detector as 1MB: the total number of unique entities supported is (8.096 * 10^9 * 0.1 / 1M ) * 3 = 2429. +For example, for a cluster with 3 data nodes, each with 8G of JVM heap size, a maximum memory percentage of 10% (default), and the entity model size of the detector as 1MB: the total number of unique entities supported is (8.096 * 10^9 * 0.1 / 1M ) * 3 = 2429. + +If you set the total number of unique entities higher than this number that you calculate (in this case: 2429), the anomaly detector makes its best effort to model the extra entities. The detector prioritizes entities that occur more often and are more recent. #### (Advanced settings) Set a shingle size @@ -111,7 +115,7 @@ Review your model configuration and select **Create detector**. ### Step 5: Observe the results -Choose the **Anomaly results** tab. You need to wait for some time to see the anomaly results. If the detector interval is 10 minutes, the detector might take more than an hour to start, as it's waiting for sufficient data to generate anomalies. +Choose the **Real-time results** or **Historical analysis** tab. For real-time results, you need to wait for some time to see the anomaly results. If the detector interval is 10 minutes, the detector might take more than an hour to start, as it's waiting for sufficient data to generate anomalies. A shorter interval means the model passes the shingle process more quickly and starts to generate the anomaly results sooner. Use the [profile detector]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/api#profile-detector) operation to make sure you have sufficient data points. @@ -123,7 +127,7 @@ If you see the detector pending in "initialization" for longer than a day, aggre Analyze anomalies with the following visualizations: - **Live anomalies** - displays live anomaly results for the last 60 intervals. For example, if the interval is 10, it shows results for the last 600 minutes. The chart refreshes every 30 seconds. -- **Anomaly history** - plots the anomaly grade with the corresponding measure of confidence. +- **Anomaly history** (for historical analysis) / **Anomaly overview** (for real-time results) - plots the anomaly grade with the corresponding measure of confidence. - **Feature breakdown** - plots the features based on the aggregation method. You can vary the date-time range of the detector. - **Anomaly occurrence** - shows the `Start time`, `End time`, `Data confidence`, and `Anomaly grade` for each detected anomaly. @@ -133,12 +137,13 @@ Analyze anomalies with the following visualizations: If you set the category field, you see an additional **Heat map** chart. The heat map correlates results for anomalous entities. This chart is empty until you select an anomalous entity. You also see the anomaly and feature line chart for the time period of the anomaly (`anomaly_grade` > 0). -Choose a filled rectangle to see a more detailed view of the anomaly. +Choose and drag over the anomaly line chart to zoom in and see a more detailed view of an anomaly. {: .note } + ### Step 4: Set up alerts -Choose **Set up alerts** and configure a monitor to notify you when anomalies are detected. For steps to create a monitor and set up notifications based on your anomaly detector, see [Monitors]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/monitors/). +Under **Real-time results**, choose **Set up alerts** and configure a monitor to notify you when anomalies are detected. For steps to create a monitor and set up notifications based on your anomaly detector, see [Monitors]({{site.url}}{{site.baseurl}}/monitoring-plugins/alerting/monitors/). If you stop or delete a detector, make sure to delete any monitors associated with it. @@ -147,7 +152,7 @@ If you stop or delete a detector, make sure to delete any monitors associated wi To see all the configuration settings for a detector, choose the **Detector configuration** tab. 1. To make any changes to the detector configuration, or fine tune the time interval to minimize any false positives, go to the **Detector configuration** section and choose **Edit**. -- You need to stop the detector to change its configuration. Confirm that you want to stop the detector and proceed. +- You need to stop a real-time or historical detector to change its configuration. Confirm that you want to stop the detector and proceed. 1. To enable or disable features, in the **Features** section, choose **Edit** and adjust the feature settings as needed. After you make your changes, choose **Save and start detector**. ### Step 8: Manage your detectors @@ -155,4 +160,4 @@ To see all the configuration settings for a detector, choose the **Detector conf To start, stop, or delete a detector, go to the **Detectors** page. 1. Choose the detector name. -2. Choose **Actions** and select **Start real-time detectors**, **Stop real-time detectors**, or **Delete detectors**. +2. Choose **Actions** and select **Start real-time detectors**, **Stop real-time detectors**, or **Delete detectors**. diff --git a/_monitoring-plugins/ad/settings.md b/_monitoring-plugins/ad/settings.md index 430bc5a0..8d3cc5b1 100644 --- a/_monitoring-plugins/ad/settings.md +++ b/_monitoring-plugins/ad/settings.md @@ -29,14 +29,15 @@ Setting | Default | Description `plugins.anomaly_detection.max_multi_entity_anomaly_detectors` | 10 | The maximum number of high cardinality detectors (with category field) in a cluster. `plugins.anomaly_detection.max_anomaly_features` | 5 | The maximum number of features for a detector. `plugins.anomaly_detection.ad_result_history_rollover_period` | 12h | How often the rollover condition is checked. If `true`, the plugin rolls over the result index to a new index. -`plugins.anomaly_detection.ad_result_history_max_docs` | 250000000 | The maximum number of documents in one result index. The plugin only counts refreshed documents in the primary shards. -`plugins.anomaly_detection.ad_result_history_retention_period` | 30d | The maximum age of the result index. If its age exceeds the threshold, the plugin deletes the rolled over result index. If the cluster has only one result index, the plugin keeps the index even if it's older than its configured retention period. -`plugins.anomaly_detection.max_entities_per_query` | 1,000 | The maximum unique values per detection interval for high cardinality detectors. By default, if the category field has more than 1,000 unique values in a detector interval, the plugin selects the top 1,000 values and orders them by `doc_count`. -`plugins.anomaly_detection.max_entities_for_preview` | 30 | The maximum unique category field values displayed with the preview operation for high cardinality detectors. If the category field has more than 30 unique values, the plugin selects the top 30 values and orders them by `doc_count`. +`plugins.anomaly_detection.ad_result_history_max_docs` | 250,000,000 | The maximum number of documents in one result index. The plugin only counts refreshed documents in the primary shards. +`plugins.anomaly_detection.ad_result_history_max_docs_per_shard` | 1,350,000,000 | The maximum number of documents in a single shard of the result index. The anomaly detection plugin only counts the refreshed documents in the primary shards. +`plugins.anomaly_detection.max_entities_per_query` | 1,000,000 | The maximum unique values per detection interval for high cardinality detectors. By default, if the category field has more than 1,000 unique values in a detector interval, the plugin selects the top 1,000 values and orders them by `doc_count`. +`plugins.anomaly_detection.max_entities_for_preview` | 5 | The maximum unique category field values displayed with the preview operation for high cardinality detectors. If the category field has more than 30 unique values, the plugin selects the top 30 values and orders them by `doc_count`. `plugins.anomaly_detection.max_primary_shards` | 10 | The maximum number of primary shards an anomaly detection index can have. `plugins.anomaly_detection.filter_by_backend_roles` | False | When you enable the security plugin and set this to `true`, the plugin filters results based on the user's backend role(s). -`plugins.anomaly_detection.max_cache_miss_handling_per_second` | 100 | High cardinality detectors use a cache to store active models. In the event of a cache miss, the cache gets the models from the model checkpoint index. Use this setting to limit the rate of fetching models. Because the thread pool for a GET operation has a queue of 1,000, we recommend setting this value below 1,000. -`plugins.anomaly_detection.max_batch_task_per_node` | 2 | Starting a historical detector triggers a batch task. This setting is the number of batch tasks that you can run per data node. You can tune this setting from 1 to 1000. If the data nodes can't support all batch tasks and you're not sure if the data nodes are capable of running more historical detectors, add more data nodes instead of changing this setting to a higher value. -`plugins.anomaly_detection.max_old_ad_task_docs_per_detector` | 10 | You can run the same historical detector many times. For each run, the anomaly detection plugin creates a new task. This setting is the number of previous tasks the plugin keeps. Set this value to at least 1 to track its last run. You can keep a maximum of 1,000 old tasks to avoid overwhelming the cluster. -`plugins.anomaly_detection.batch_task_piece_size` | 1000 | The date range for a historical task is split into smaller pieces and the anomaly detection plugin runs the task piece by piece. Each piece contains 1,000 detection intervals by default. For example, if detector interval is 1 minute and one piece is 1000 minutes, the feature data is queried every 1,000 minutes. You can change this setting from 1 to 10,000. +`plugins.anomaly_detection.max_batch_task_per_node` | 10 | Starting a historical detector triggers a batch task. This setting is the number of batch tasks that you can run per data node. You can tune this setting from 1 to 1000. If the data nodes can't support all batch tasks and you're not sure if the data nodes are capable of running more historical detectors, add more data nodes instead of changing this setting to a higher value. +`plugins.anomaly_detection.max_old_ad_task_docs_per_detector` | 1 | You can run the same historical detector many times. For each run, the anomaly detection plugin creates a new task. This setting is the number of previous tasks the plugin keeps. Set this value to at least 1 to track its last run. You can keep a maximum of 1,000 old tasks to avoid overwhelming the cluster. +`plugins.anomaly_detection.batch_task_piece_size` | 1,000 | The date range for a historical task is split into smaller pieces and the anomaly detection plugin runs the task piece by piece. Each piece contains 1,000 detection intervals by default. For example, if detector interval is 1 minute and one piece is 1000 minutes, the feature data is queried every 1,000 minutes. You can change this setting from 1 to 10,000. `plugins.anomaly_detection.batch_task_piece_interval_seconds` | 5 | Add a time interval between historical detector tasks. This interval prevents the task from consuming too much of the available resources and starving other operations like search and bulk index. You can change this setting from 1 to 600 seconds. +`plugins.anomaly_detection.max_top_entities_for_historical_analysis` | 1,000 | The maximum number of top entities that you run for a high-cardinality detector historical analysis. +`plugins.anomaly_detection.max_running_entities_per_detector_for_historical_analysis` | 10 | How many entity tasks you can run in parallel for one HC detector. The cluster availble task slots will impact how many entities can run in parallel as well. For example, the cluster has 3 data nodes, each data node has 10 task slots by default. But if we have already started 2 HC detectors and each HC running 10 entities, and start a single-flow detector which takes 1 task slot, then the availabe task slots will be 10 * 3 - 10 * 2 - 1 = 9. Then, if we start a new HC detector, it can only run 9 entities in parallel, not 10. From c2f1018b3685c6ce7f41135f8b237cd08fa0ffe4 Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Sun, 3 Oct 2021 10:50:46 -0700 Subject: [PATCH 053/229] incorporated feedback --- _upgrade-to/upgrade-to.md | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/_upgrade-to/upgrade-to.md b/_upgrade-to/upgrade-to.md index 0d554370..cd403f7e 100644 --- a/_upgrade-to/upgrade-to.md +++ b/_upgrade-to/upgrade-to.md @@ -221,18 +221,37 @@ Check [Upgrade paths]({{site.url}}{{site.baseurl}}/upgrade-to/upgrade-to/#upgrad 1. Make sure the following environment variables are set: - `ES_HOME` - Path to the existing Elasticsearch installation home. + + ```bash + export ES_HOME = /home/workspace/upgrade-demo/node1/elasticsearch-7.10.2 + ``` + - `ES_PATH_CONF` - Path to the existing Elasticsearch config directory. + + ```bash + export ES_PATH_CONF = /home/workspace/upgrade-demo/node1/os-config + ``` + - `OPENSEARCH_HOME` - Path to the OpenSearch installation home. + + ```bash + export OPENSEARCH_HOME = /home/workspace/upgrade-demo/node1/opensearch-1.0.0 + ``` + - `OPENSEARCH_PATH_CONF` - Path to the OpenSearch config directory. + ```bash + export OPENSEARCH_PATH_CONF = /home/workspace/upgrade-demo/node1/opensearch-config + ``` + 1. The `opensearch-upgrade` tool is in the `bin` directory of the distribution. Run the following command from the distribution home: - Make sure you run this tool as the same user running the current Elasticsearch service. - {: .note } + Make sure you run this tool as the same user running the current Elasticsearch service. + {: .note } - ```json - ./bin/opensearch-upgrade - ``` + ```json + ./bin/opensearch-upgrade + ``` 1. Stop Elasticsearch OSS on the node. @@ -270,6 +289,6 @@ Behind the scenes, the `opensearch-upgrade` tool performs the following tasks in 1. Looks for a valid Elasticsearch installation on the current node. After it finds the installation, it reads the `elasticsearch.yml` file to get the endpoint details and connects to the locally running Elasticsearch service. If the tool can't find an Elasticsearch installation, it tries to get the path from the `ES_HOME` location. 1. Verifies if the existing version of Elasticsearch is compatible with the OpenSearch version. It prints a summary of the information gathered to the console and prompts you for a confirmation to proceed. 1. Imports the settings from the `elasticsearch.yml` config file into the `opensearch.yml` config file. -1. Copies across any custom JVM options from the `$ES_PATH_CONF/jvm.options.d` directory into the `$OPENSEARCH_PATH_CONF/jvm.options.d` directory . Similarly, it also imports the logging configurations from the `$ES_PATH_CONF/log4j2.properties` file into the `$OPENSEARCH_PATH_CONF/log4j2.properties` file. +1. Copies across any custom JVM options from the `$ES_PATH_CONF/jvm.options.d` directory into the `$OPENSEARCH_PATH_CONF/jvm.options.d` directory. Similarly, it also imports the logging configurations from the `$ES_PATH_CONF/log4j2.properties` file into the `$OPENSEARCH_PATH_CONF/log4j2.properties` file. 1. Installs the core plugins that you’ve currently installed in the `$ES_HOME/plugins` directory. You must install all other third-party community plugins manually. 1. Imports the secure settings from the `elasticsearch.keystore` file (if any) into the `opensearch.keystore` file. If the keystore file is password protected, the `opensearch-upgrade` tool prompts you to enter the password. From 370ff0fc57475e4cd255039efffef90f8afbf2c4 Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Sun, 3 Oct 2021 10:57:58 -0700 Subject: [PATCH 054/229] spacing --- _upgrade-to/upgrade-to.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/_upgrade-to/upgrade-to.md b/_upgrade-to/upgrade-to.md index cd403f7e..b21ec89c 100644 --- a/_upgrade-to/upgrade-to.md +++ b/_upgrade-to/upgrade-to.md @@ -265,22 +265,22 @@ Check [Upgrade paths]({{site.url}}{{site.baseurl}}/upgrade-to/upgrade-to/#upgrad 1. Start OpenSearch on the node: - ```json - ./bin/opensearch -d. - ``` + ```json + ./bin/opensearch -d. + ``` 1. Repeat steps 2--6 until all nodes are using the new version. 1. After all nodes are using the new version, re-enable shard allocation: - ```json - PUT _cluster/settings - { - "persistent": { - "cluster.routing.allocation.enable": "all" - } + ```json + PUT _cluster/settings + { + "persistent": { + "cluster.routing.allocation.enable": "all" } - ``` + } + ``` ### How it works From 19fb0d0948ac3c7960c8bf7540419b13484e2b39 Mon Sep 17 00:00:00 2001 From: Omurbek Date: Mon, 4 Oct 2021 12:30:23 +0300 Subject: [PATCH 055/229] Update policies.md --- _im-plugin/ism/policies.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/_im-plugin/ism/policies.md b/_im-plugin/ism/policies.md index afad9396..29ac228b 100644 --- a/_im-plugin/ism/policies.md +++ b/_im-plugin/ism/policies.md @@ -560,7 +560,7 @@ The following sample template policy is for a rollover use case. "index_patterns": ["log*"], "template": { "settings": { - "opendistro.index_state_management.rollover_alias": "log" + "plugins.index_state_management.rollover_alias": "log" } } } @@ -588,12 +588,6 @@ The following sample template policy is for a rollover use case. } ``` -5. Verify if the policy is attached to the `log-000001` index: - - ```json - GET _opendistro/_ism/explain/log-000001?pretty - ``` - ## Example policy The following example policy implements a `hot`, `warm`, and `delete` workflow. You can use this policy as a template to prioritize resources to your indices based on their levels of activity. From 164ead7dbd88dea24b6753307ed85ecfca93a3ea Mon Sep 17 00:00:00 2001 From: Omurbek Date: Mon, 4 Oct 2021 12:39:06 +0300 Subject: [PATCH 056/229] Update policies.md --- _im-plugin/ism/policies.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/_im-plugin/ism/policies.md b/_im-plugin/ism/policies.md index 29ac228b..cc09eab1 100644 --- a/_im-plugin/ism/policies.md +++ b/_im-plugin/ism/policies.md @@ -588,6 +588,12 @@ The following sample template policy is for a rollover use case. } ``` +5. Verify if the policy is attached to the `log-000001` index: + + ```json + GET _plugins/_ism/explain/log-000001?pretty + ``` + ## Example policy The following example policy implements a `hot`, `warm`, and `delete` workflow. You can use this policy as a template to prioritize resources to your indices based on their levels of activity. From a3a1795ab5ff183067f9c1c34d599fbc1521b8b4 Mon Sep 17 00:00:00 2001 From: Maarten-Jan <31125767+maartenjanvangool@users.noreply.github.com> Date: Mon, 4 Oct 2021 17:00:43 +0200 Subject: [PATCH 057/229] Update java-rest-high-level.md The file has a compile error, I think ); is missing --- _clients/java-rest-high-level.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_clients/java-rest-high-level.md b/_clients/java-rest-high-level.md index 892f7341..314725f4 100644 --- a/_clients/java-rest-high-level.md +++ b/_clients/java-rest-high-level.md @@ -93,7 +93,7 @@ public class RESTClientSample { HashMap mapping = new HashMap(); mapping.put("properties", ageMapping); createIndexRequest.mapping(mapping); - CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest, RequestOptions.DEFAULT + CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest, RequestOptions.DEFAULT); //Adding data to the index. IndexRequest request = new IndexRequest("custom-index"); //Add a document to the custom-index we created. From 790459778cafd87ea854f5b1c96e855acfb914f3 Mon Sep 17 00:00:00 2001 From: keithhc2 Date: Mon, 4 Oct 2021 10:03:32 -0700 Subject: [PATCH 058/229] A comma --- _im-plugin/ism/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_im-plugin/ism/index.md b/_im-plugin/ism/index.md index 0104dc68..4202b849 100644 --- a/_im-plugin/ism/index.md +++ b/_im-plugin/ism/index.md @@ -31,7 +31,7 @@ To get started, choose **Index Management** in OpenSearch Dashboards. A policy is a set of rules that describes how an index should be managed. For information about creating a policy, see [Policies]({{site.url}}{{site.baseurl}}/im-plugin/ism/policies/). -You can use the JSON editor or visual editor to create policies. Compared to the JSON editor, the visual editor offers a more structured way of defining policies by separating the process into creating error notifications, defining ISM templates, and adding states. We recommend using the visual editor if you want to see pre-defined fields such as which actions you can assign to a state or under what conditions a state can transition into a destination state. +You can use the JSON editor or visual editor to create policies. Compared to the JSON editor, the visual editor offers a more structured way of defining policies by separating the process into creating error notifications, defining ISM templates, and adding states. We recommend using the visual editor if you want to see pre-defined fields, such as which actions you can assign to a state or under what conditions a state can transition into a destination state. #### JSON editor From e6403d89a40eff7436a800288b0887b53e2fe25e Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Mon, 4 Oct 2021 12:08:00 -0700 Subject: [PATCH 059/229] ad changes for 1.1 --- _monitoring-plugins/ad/api.md | 1807 ++++++++++------------------ _monitoring-plugins/ad/index.md | 13 +- _monitoring-plugins/ad/settings.md | 23 +- 3 files changed, 657 insertions(+), 1186 deletions(-) diff --git a/_monitoring-plugins/ad/api.md b/_monitoring-plugins/ad/api.md index 2a8427e3..e3db5aac 100644 --- a/_monitoring-plugins/ad/api.md +++ b/_monitoring-plugins/ad/api.md @@ -24,7 +24,7 @@ Introduced 1.0 Creates an anomaly detector. -This command creates a detector named `http_requests` that finds anomalies based on the sum and average number of failed HTTP requests: +This command creates a detector named `test-detector` that finds anomalies based on the sum of the `value` field: #### Request @@ -266,7 +266,6 @@ Passes a date range to the anomaly detector to return any anomalies within that ```json POST _plugins/_anomaly_detection/detectors/_preview - { "period_start": 1612982516000, "period_end": 1614278539000, @@ -426,7 +425,6 @@ If you specify a category field, each result is associated with an entity: ] } ... - ``` Or, you can specify the detector ID: @@ -434,9 +432,9 @@ Or, you can specify the detector ID: ```json POST _plugins/_anomaly_detection/detectors/_preview { - "detector_id": "sYkUvHcBiZv51f-Lv8QN", - "period_start": 1612982516000, - "period_end": 1614278539000 + "detector_id": "sYkUvHcBiZv51f-Lv8QN", + "period_start": 1612982516000, + "period_end": 1614278539000 } ``` @@ -471,8 +469,8 @@ To start historical analysis: ```json POST _plugins/_anomaly_detection/detectors//_start { - "start_time": 1503168590000, - "end_time": 1617301324000 + "start_time": 1503168590000, + "end_time": 1617301324000 } ``` @@ -498,10 +496,11 @@ Stopped detector: m4ccEnIBTXsGi3mvMt9p To stop historical analysis: -```jsom +```json POST _plugins/_anomaly_detection/detectors//_stop?historical=true ``` + --- ## Search detector result @@ -515,7 +514,6 @@ Returns all results for a search query. ```json GET _plugins/_anomaly_detection/detectors/results/_search POST _plugins/_anomaly_detection/detectors/results/_search - { "query": { "bool": { @@ -552,7 +550,7 @@ POST _plugins/_anomaly_detection/detectors/results/_search "max_score": 1, "hits": [ { - "_index": ".opendistro-anomaly-results-history-2020.04.30-1", + "_index": ".opensearch-anomaly-results-history-2020.04.30-1", "_type": "_doc", "_id": "_KBrzXEBbpoKkFM5mStm", "_version": 1, @@ -578,7 +576,7 @@ POST _plugins/_anomaly_detection/detectors/results/_search } }, { - "_index": ".opendistro-anomaly-results-history-2020.04.30-1", + "_index": ".opensearch-anomaly-results-history-2020.04.30-1", "_type": "_doc", "_id": "EqB1zXEBbpoKkFM5qyyE", "_version": 1, @@ -683,7 +681,7 @@ POST _plugins/_anomaly_detection/detectors/results/_search "max_score": null, "hits": [ { - "_index": ".opendistro-anomaly-results-history-2020.11.07-1", + "_index": ".opensearch-anomaly-results-history-2020.11.07-1", "_type": "_doc", "_id": "BiItoHUBTpMGN-4KARY5", "_version": 1, @@ -719,7 +717,7 @@ POST _plugins/_anomaly_detection/detectors/results/_search ] }, { - "_index": ".opendistro-anomaly-results-history-2020.11.07-1", + "_index": ".opensearch-anomaly-results-history-2020.11.07-1", "_type": "_doc", "_id": "wiImoHUBTpMGN-4KlhXs", "_version": 1, @@ -755,7 +753,7 @@ POST _plugins/_anomaly_detection/detectors/results/_search ] }, { - "_index": ".opendistro-anomaly-results-history-2020.11.07-1", + "_index": ".opensearch-anomaly-results-history-2020.11.07-1", "_type": "_doc", "_id": "ZiIcoHUBTpMGN-4KhhVA", "_version": 1, @@ -832,7 +830,7 @@ GET _plugins/_anomaly_detection/detectors/results/_search "max_score": 2.1366, "hits": [ { - "_index": ".opendistro-anomaly-detection-state", + "_index": ".opensearch-anomaly-detection-state", "_type": "_doc", "_id": "CoM8WncBtt2qvI-LZO7_", "_version": 8, @@ -943,10 +941,10 @@ You can specify the following options. Options | Description | Type | Required :--- | :--- |:--- |:--- | -`anomalyThreshold` | Filter out low anomaly grade results. Default is -1. The lowest possible anomaly grade is 0. -1 means that the detector returns all results. | `float` | No -`dateRangeFilter` | Specify the date range in:
- `startTime` (int): Start time to collect results. Recorded in milliseconds since the Unix Epoch.
- `endTime` (int): End time to collect results. Recorded in milliseconds since the Unix Epoch.
- `fieldName` (string): The field that you want to match the start and end time. | `object` | Yes -`entity` | If not empty, the parameter contains the entity name and value. Default is empty.
- `name` (string): Field name that you want to search in.
- `value` (string): Entity value that you want to search for. | `object` | No -`sort` | If not empty, sorts the result by a field in a certain order. Default is empty. Properties of `sort`:
- `direction` (string): Specify "desc" or "asc" for descending or ascending order.
- `field` (string): Order the results by a field. | `object` | No +`anomalyThreshold` | Specify a threshold to filter out low anomaly grade results. Default is -1. Because the lowest possible anomaly grade is 0, -1 means that the detector returns all results. | `float` | No +`dateRangeFilter` | Specify the date range in:
- `startTime` (int): Start time to collect results. Add in milliseconds since the Unix Epoch.
- `endTime` (int): End time to collect results. Add in milliseconds since the Unix Epoch.
- `fieldName` (string): The field that you want to match the start and end time. | `object` | Yes +`entity` | Specify the entity name and value. Default is empty.
- `name` (string): Field name that you want to search in.
- `value` (string): Entity value that you want to search for. | `object` | No +`sort` | Sort the result by a field in a certain order. Default is empty. Properties of `sort`:
- `direction` (string): Specify "desc" or "asc" for descending or ascending order.
- `field` (string): Order the results by a field. | `object` | No --- @@ -996,127 +994,127 @@ POST _plugins/_anomaly_detection/detectors/tasks/_search ```json { - "took" : 5, - "timed_out" : false, - "_shards" : { - "total" : 1, - "successful" : 1, - "skipped" : 0, - "failed" : 0 + "took": 5, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 }, - "hits" : { - "total" : { - "value" : 1, - "relation" : "eq" + "hits": { + "total": { + "value": 1, + "relation": "eq" }, - "max_score" : null, - "hits" : [ + "max_score": null, + "hits": [ { - "_index" : ".opendistro-anomaly-detection-state", - "_type" : "_doc", - "_id" : "TM3tOHwBCi2h__AOXlyQ", - "_version" : 3, - "_seq_no" : 14, - "_primary_term" : 1, - "_score" : null, - "_source" : { - "detector_id" : "rlDtOHwBD5tpxlbyW7Nt", - "error" : "", - "detection_date_range" : { - "start_time" : 1632437852100, - "end_time" : 1633042652100 + "_index": ".opensearch-anomaly-detection-state", + "_type": "_doc", + "_id": "TM3tOHwBCi2h__AOXlyQ", + "_version": 3, + "_seq_no": 14, + "_primary_term": 1, + "_score": null, + "_source": { + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "error": "", + "detection_date_range": { + "start_time": 1632437852100, + "end_time": 1633042652100 }, - "task_progress" : 0.5, - "last_update_time" : 1633042667358, - "execution_start_time" : 1633042652810, - "state" : "RUNNING", - "coordinating_node" : "2hEGbUw6ShaiKe05n_xLdA", - "task_type" : "HISTORICAL_HC_DETECTOR", - "started_by" : "admin", - "init_progress" : 0.0, - "is_latest" : true, - "detector" : { - "category_field" : [ + "task_progress": 0.5, + "last_update_time": 1633042667358, + "execution_start_time": 1633042652810, + "state": "RUNNING", + "coordinating_node": "2hEGbUw6ShaiKe05n_xLdA", + "task_type": "HISTORICAL_HC_DETECTOR", + "started_by": "admin", + "init_progress": 0, + "is_latest": true, + "detector": { + "category_field": [ "type" ], - "description" : "test", - "ui_metadata" : { - "features" : { - "test-feature" : { - "aggregationBy" : "sum", - "aggregationOf" : "value", - "featureType" : "simple_aggs" + "description": "test", + "ui_metadata": { + "features": { + "test-feature": { + "aggregationBy": "sum", + "aggregationOf": "value", + "featureType": "simple_aggs" } }, - "filters" : [ ] + "filters": [] }, - "feature_attributes" : [ + "feature_attributes": [ { - "feature_id" : "7VDtOHwBD5tpxlbyWqPs", - "feature_enabled" : true, - "feature_name" : "test-feature", - "aggregation_query" : { - "test_feature" : { - "sum" : { - "field" : "value" + "feature_id": "7VDtOHwBD5tpxlbyWqPs", + "feature_enabled": true, + "feature_name": "test-feature", + "aggregation_query": { + "test_feature": { + "sum": { + "field": "value" } } } } ], - "schema_version" : 0, - "time_field" : "timestamp", - "last_update_time" : 1633042652012, - "indices" : [ + "schema_version": 0, + "time_field": "timestamp", + "last_update_time": 1633042652012, + "indices": [ "server_log" ], - "window_delay" : { - "period" : { - "unit" : "Minutes", - "interval" : 1 + "window_delay": { + "period": { + "unit": "Minutes", + "interval": 1 } }, - "detection_interval" : { - "period" : { - "unit" : "Minutes", - "interval" : 5 + "detection_interval": { + "period": { + "unit": "Minutes", + "interval": 5 } }, - "name" : "test-detector", - "filter_query" : { - "match_all" : { - "boost" : 1.0 + "name": "test-detector", + "filter_query": { + "match_all": { + "boost": 1 } }, - "shingle_size" : 8, - "user" : { - "backend_roles" : [ + "shingle_size": 8, + "user": { + "backend_roles": [ "admin" ], - "custom_attribute_names" : [ ], - "roles" : [ + "custom_attribute_names": [], + "roles": [ "own_index", "all_access" ], - "name" : "admin", - "user_requested_tenant" : null + "name": "admin", + "user_requested_tenant": null }, - "detector_type" : "MULTI_ENTITY" + "detector_type": "MULTI_ENTITY" }, - "user" : { - "backend_roles" : [ + "user": { + "backend_roles": [ "admin" ], - "custom_attribute_names" : [ ], - "roles" : [ + "custom_attribute_names": [], + "roles": [ "own_index", "all_access" ], - "name" : "admin", - "user_requested_tenant" : "__user__" + "name": "admin", + "user_requested_tenant": "__user__" } }, - "sort" : [ + "sort": [ 1633042652810 ] } @@ -1146,19 +1144,19 @@ DELETE _plugins/_anomaly_detection/detectors/ ```json { - "_index" : ".opendistro-anomaly-detectors", - "_type" : "_doc", - "_id" : "m4ccEnIBTXsGi3mvMt9p", - "_version" : 2, - "result" : "deleted", - "forced_refresh" : true, - "_shards" : { - "total" : 2, - "successful" : 2, - "failed" : 0 + "_index": ".opensearch-anomaly-detectors", + "_type": "_doc", + "_id": "m4ccEnIBTXsGi3mvMt9p", + "_version": 2, + "result": "deleted", + "forced_refresh": true, + "_shards": { + "total": 2, + "successful": 2, + "failed": 0 }, - "_seq_no" : 6, - "_primary_term" : 1 + "_seq_no": 6, + "_primary_term": 1 } ``` @@ -1168,13 +1166,12 @@ DELETE _plugins/_anomaly_detection/detectors/ Introduced 1.1 {: .label .label-purple } -Deletes a detector results based on a query. +Deletes the results of a detector based on a query. #### Request ```json DELETE _plugins/_anomaly_detection/detectors/results - { "query": { "bool": { @@ -1211,23 +1208,23 @@ DELETE _plugins/_anomaly_detection/detectors/results ```json { - "took" : 48, - "timed_out" : false, - "total" : 28, - "updated" : 0, - "created" : 0, - "deleted" : 28, - "batches" : 1, - "version_conflicts" : 0, - "noops" : 0, - "retries" : { - "bulk" : 0, - "search" : 0 + "took": 48, + "timed_out": false, + "total": 28, + "updated": 0, + "created": 0, + "deleted": 28, + "batches": 1, + "version_conflicts": 0, + "noops": 0, + "retries": { + "bulk": 0, + "search": 0 }, - "throttled_millis" : 0, - "requests_per_second" : -1.0, - "throttled_until_millis" : 0, - "failures" : [ ] + "throttled_millis": 0, + "requests_per_second": -1, + "throttled_until_millis": 0, + "failures": [] } ``` @@ -1237,7 +1234,7 @@ DELETE _plugins/_anomaly_detection/detectors/results Introduced 1.1 {: .label .label-purple } -Validates detector before creating. This API shows you any invalid fields in your configuration and also recommendations on how to fix it. +Validates detector before creating. Lists any invalid fields in your configuration and recommends ways to fix these invalid fields. #### Request @@ -1312,8 +1309,7 @@ POST _plugins/_anomaly_detection/detectors/_validate/detector,model "message": "should be set|not_exist|empty" }, "feature_attributes": { - // exist when message is "there exists non-numeric field" - [Optional]"problematic_feature_name1": { + "problematic_feature_name1": { "message": "{field} is invalid with {exceptionMessage}" } "problematic_feature_name2": { @@ -1336,8 +1332,7 @@ POST _plugins/_anomaly_detection/detectors/_validate/detector,model "message": "data is too sparse after filter_query is applied" }, "detection_interval": { - // exists when suggested value can be found - [Optional]"suggested_value": { + "suggested_value": { "period": { "interval": 1, "unit": "Minutes" @@ -1361,8 +1356,7 @@ POST _plugins/_anomaly_detection/detectors/_validate/detector,model "message": "model size exceeds memory limit, please stop/delete unused detectors, or reduce shingle size or number of features" }, "window_delay": { - // exists when suggested value can be found - [Optional]"suggested_value": { + "suggested_value": { "period": { "interval": 1, "unit": "Minutes" @@ -1372,7 +1366,6 @@ POST _plugins/_anomaly_detection/detectors/_validate/detector,model } } } - ``` --- @@ -1586,133 +1579,133 @@ GET _plugins/_anomaly_detection/detectors/?job=true ```json { - "_id": "LJxGsXcBoDQA8W1Q--A1", - "_version": 1, - "_primary_term": 1, - "_seq_no": 0, - "anomaly_detector": { - "name": "test2", - "description": "test", - "time_field": "timestamp", - "indices": [ - "server_log" - ], - "filter_query": { - "match_all": { - "boost": 1.0 - } - }, - "detection_interval": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "K5xGsXcBoDQA8W1Q-uCF", - "feature_name": "F1", - "feature_enabled": *true*, - "aggregation_query": { - "f_1": { - "sum": { - "field": "value" - } - } - } - } - ], - "last_update_time": 1613586955060, - "detector_type": "MULTI_ENTITY" + "_id": "LJxGsXcBoDQA8W1Q--A1", + "_version": 1, + "_primary_term": 1, + "_seq_no": 0, + "anomaly_detector": { + "name": "test2", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" + ], + "filter_query": { + "match_all": { + "boost": 1 + } }, - "anomaly_detector_job": { - "name": "LJxGsXcBoDQA8W1Q--A1", - "schedule": { - "interval": { - "start_time": 1613587220387, - "period": 1, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "enabled": *false*, - "enabled_time": 1613587220387, - "last_update_time": 1613587289169, - "lock_duration_seconds": 60, - "disabled_time": 1613587289169 + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" + } }, - "anomaly_detection_task": { - "task_id": "WZ5LsXcBoDQA8W1QmUa3", - "last_update_time": 1613587349022, - "error": "Task cancelled by user", - "state": "STOPPED", - "detector_id": "LJxGsXcBoDQA8W1Q--A1", - "task_progress": 0.26321793, - "init_progress": 1.0, - "current_piece": 1611030900000, - "execution_start_time": 1613587257783, - "execution_end_time": 1613587349022, - "is_latest": *true*, - "task_type": "HISTORICAL", - "coordinating_node": "NSw5j-3YQeGkH8KESVKlzw", - "worker_node": "NSw5j-3YQeGkH8KESVKlzw", - "detector": { - "name": "test2", - "description": "test", - "time_field": "timestamp", - "indices": [ - "server_log" - ], - "filter_query": { - "match_all": { - "boost": 1.0 - } - }, - "detection_interval": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "K5xGsXcBoDQA8W1Q-uCF", - "feature_name": "F1", - "feature_enabled": *true*, - "aggregation_query": { - "f_1": { - "sum": { - "field": "value" - } - } - } - } - ], - "last_update_time": 1613586955060, - "detector_type": "MULTI_ENTITY" + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "K5xGsXcBoDQA8W1Q-uCF", + "feature_name": "F1", + "feature_enabled": "true", + "aggregation_query": { + "f_1": { + "sum": { + "field": "value" + } + } } + } + ], + "last_update_time": 1613586955060, + "detector_type": "MULTI_ENTITY" + }, + "anomaly_detector_job": { + "name": "LJxGsXcBoDQA8W1Q--A1", + "schedule": { + "interval": { + "start_time": 1613587220387, + "period": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "enabled": "false", + "enabled_time": 1613587220387, + "last_update_time": 1613587289169, + "lock_duration_seconds": 60, + "disabled_time": 1613587289169 + }, + "anomaly_detection_task": { + "task_id": "WZ5LsXcBoDQA8W1QmUa3", + "last_update_time": 1613587349022, + "error": "Task cancelled by user", + "state": "STOPPED", + "detector_id": "LJxGsXcBoDQA8W1Q--A1", + "task_progress": 0.26321793, + "init_progress": 1, + "current_piece": 1611030900000, + "execution_start_time": 1613587257783, + "execution_end_time": 1613587349022, + "is_latest": "true", + "task_type": "HISTORICAL", + "coordinating_node": "NSw5j-3YQeGkH8KESVKlzw", + "worker_node": "NSw5j-3YQeGkH8KESVKlzw", + "detector": { + "name": "test2", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" + ], + "filter_query": { + "match_all": { + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "K5xGsXcBoDQA8W1Q-uCF", + "feature_name": "F1", + "feature_enabled": "true", + "aggregation_query": { + "f_1": { + "sum": { + "field": "value" + } + } + } + } + ], + "last_update_time": 1613586955060, + "detector_type": "MULTI_ENTITY" } + } } ``` @@ -1912,9 +1905,6 @@ Returns all anomaly detectors for a search query. ```json GET _plugins/_anomaly_detection/detectors/_search POST _plugins/_anomaly_detection/detectors/_search - -Sample Input: - { "query": { "bool": { @@ -1936,70 +1926,62 @@ Sample Input: ```json { - "took": 13, + "took": 2, "timed_out": false, "_shards": { - "total": 5, - "successful": 5, + "total": 1, + "successful": 1, "skipped": 0, "failed": 0 }, "hits": { "total": { - "value": 994, + "value": 1, "relation": "eq" }, - "max_score": 3.5410638, + "max_score": 0, "hits": [ { - "_index": ".opendistro-anomaly-detectors", + "_index": ".opensearch-anomaly-detectors", "_type": "_doc", - "_id": "m4ccEnIBTXsGi3mvMt9p", - "_version": 2, - "_seq_no": 221, + "_id": "rlDtOHwBD5tpxlbyW7Nt", + "_version": 1, + "_seq_no": 0, "_primary_term": 1, - "_score": 3.5410638, + "_score": 0, "_source": { "name": "test-detector", - "description": "Test detector", + "description": "test", "time_field": "timestamp", "indices": [ - "order*" + "server_log" ], "filter_query": { - "bool": { - "filter": [ - { - "exists": { - "field": "value", - "boost": 1 - } - } - ], - "adjust_pure_negative": true, + "match_all": { "boost": 1 } }, "detection_interval": { "period": { - "interval": 10, - "unit": "MINUTES" + "interval": 5, + "unit": "Minutes" } }, "window_delay": { "period": { "interval": 1, - "unit": "MINUTES" + "unit": "Minutes" } }, + "shingle_size": 8, "schema_version": 0, "feature_attributes": [ { - "feature_id": "xxokEnIBcpeWMD987A1X", - "feature_name": "total_order", + "feature_id": "7VDtOHwBD5tpxlbyWqPs", + "feature_name": "test-feature", "feature_enabled": true, "aggregation_query": { - "total_order": { + "test_feature": { "sum": { "field": "value" } @@ -2007,7 +1989,23 @@ Sample Input: } } ], - "last_update_time": 1589442309241 + "last_update_time": 1633042652012, + "category_field": [ + "type" + ], + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": null + }, + "detector_type": "MULTI_ENTITY" } } ] @@ -2065,261 +2063,6 @@ GET _plugins/_anomaly_detection/stats/ } ] }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578975, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_k2gq2eRP0vTV2LNNyFdIqg", - "last_checkpoint_time": 1629827339733, - "entity": [ - { - "name": "host", - "value": "server_1" - }, - { - "name": "service", - "value": "app_0" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578980, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_DOze7d0HnK3K54g3Emk1XA", - "last_checkpoint_time": 1629827343186, - "entity": [ - { - "name": "host", - "value": "server_2" - }, - { - "name": "service", - "value": "app_3" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578977, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_NgGmglQvOMQQciDdPxN_Ig", - "last_checkpoint_time": 1629827340961, - "entity": [ - { - "name": "host", - "value": "server_3" - }, - { - "name": "service", - "value": "app_5" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578977, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_oTdcenY1L5bqa6chUxg7xw", - "last_checkpoint_time": 1629827340263, - "entity": [ - { - "name": "host", - "value": "server_1" - }, - { - "name": "service", - "value": "app_1" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578979, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_zxSqAWv5Iz19v-Hnqhrwrw", - "last_checkpoint_time": 1629827342814, - "entity": [ - { - "name": "host", - "value": "server_1" - }, - { - "name": "service", - "value": "app_5" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578976, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_Yu8X2rA39lhjYzqebjLxhQ", - "last_checkpoint_time": 1629827339992, - "entity": [ - { - "name": "host", - "value": "server_2" - }, - { - "name": "service", - "value": "app_1" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578978, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_6SvF11RCqf7HYbY56BnFKA", - "last_checkpoint_time": 1629827341806, - "entity": [ - { - "name": "host", - "value": "server_3" - }, - { - "name": "service", - "value": "app_2" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578980, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_3S8k6q_DLFhw3hboko3dfw", - "last_checkpoint_time": 1629827343371, - "entity": [ - { - "name": "host", - "value": "server_1" - }, - { - "name": "service", - "value": "app_3" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578978, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_0uafBokvEYuncGbjP3D2qA", - "last_checkpoint_time": 1629827342302, - "entity": [ - { - "name": "host", - "value": "server_2" - }, - { - "name": "service", - "value": "app_5" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578977, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_93DEK2PooWlHF6gkh-0hIA", - "last_checkpoint_time": 1629827340727, - "entity": [ - { - "name": "host", - "value": "server_2" - }, - { - "name": "service", - "value": "app_4" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578975, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_FuqXh0HBXlPhKepOc6JADQ", - "last_checkpoint_time": 1629827338908, - "entity": [ - { - "name": "host", - "value": "server_3" - }, - { - "name": "service", - "value": "app_6" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578979, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_B4zrbSQ1-pvdBLx0FzQxvw", - "last_checkpoint_time": 1629827342611, - "entity": [ - { - "name": "host", - "value": "server_3" - }, - { - "name": "service", - "value": "app_3" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578978, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_xm_gKBMKlgymKcoqZyXT8A", - "last_checkpoint_time": 1629827341365, - "entity": [ - { - "name": "host", - "value": "server_2" - }, - { - "name": "service", - "value": "app_0" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578978, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_OnZ4CP-yJF5llO57gUjM6w", - "last_checkpoint_time": 1629827341599, - "entity": [ - { - "name": "host", - "value": "server_3" - }, - { - "name": "service", - "value": "app_1" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578977, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_E_uWreoeJpGrAMMaitg8BA", - "last_checkpoint_time": 1629827340418, - "entity": [ - { - "name": "host", - "value": "server_3" - }, - { - "name": "service", - "value": "app_4" - } - ] - }, { "detector_id": "mmZFeXsB7JcKN0mdnMf4", "model_type": "entity", @@ -2420,7 +2163,7 @@ GET _plugins/_anomaly_detection/stats/ ``` The `model_count` parameter shows the total number of models running on each node’s memory. -Historical detectors contain additional fields: +Historical detectors contain the following additional fields: - `ad_total_batch_task_execution_count` - `ad_executing_batch_task_count` @@ -2469,71 +2212,6 @@ Historical detectors contain additional fields: } ] }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152729, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error21", - "last_checkpoint_time": 1633043555143, - "entity": [ - { - "name": "type", - "value": "error21" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152727, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error13", - "last_checkpoint_time": 1633043554046, - "entity": [ - { - "name": "type", - "value": "error13" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152753, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error24", - "last_checkpoint_time": 1633043853986, - "entity": [ - { - "name": "type", - "value": "error24" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152792, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error25", - "last_checkpoint_time": 1633043857320, - "entity": [ - { - "name": "type", - "value": "error25" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152779, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error28", - "last_checkpoint_time": 1633043856244, - "entity": [ - { - "name": "type", - "value": "error28" - } - ] - }, { "detector_id": "rlDtOHwBD5tpxlbyW7Nt", "model_type": "entity", @@ -2599,97 +2277,6 @@ Historical detectors contain additional fields: } ] }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152750, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error32", - "last_checkpoint_time": 1633043854080, - "entity": [ - { - "name": "type", - "value": "error32" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152784, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error23", - "last_checkpoint_time": 1633043857463, - "entity": [ - { - "name": "type", - "value": "error23" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152774, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error26", - "last_checkpoint_time": 1633043856308, - "entity": [ - { - "name": "type", - "value": "error26" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152734, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error14", - "last_checkpoint_time": 1633043555939, - "entity": [ - { - "name": "type", - "value": "error14" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152731, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error9", - "last_checkpoint_time": 1633043257214, - "entity": [ - { - "name": "type", - "value": "error9" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152730, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error19", - "last_checkpoint_time": 1633043553882, - "entity": [ - { - "name": "type", - "value": "error19" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152732, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error18", - "last_checkpoint_time": 1633043554874, - "entity": [ - { - "name": "type", - "value": "error18" - } - ] - }, { "detector_id": "rlDtOHwBD5tpxlbyW7Nt", "model_type": "entity", @@ -2729,123 +2316,6 @@ Historical detectors contain additional fields: } ] }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152711, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error6", - "last_checkpoint_time": 1633043254281, - "entity": [ - { - "name": "type", - "value": "error6" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152716, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error4", - "last_checkpoint_time": 1633043257797, - "entity": [ - { - "name": "type", - "value": "error4" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152709, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error2", - "last_checkpoint_time": 1633043260938, - "entity": [ - { - "name": "type", - "value": "error2" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152742, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error30", - "last_checkpoint_time": 1633043853983, - "entity": [ - { - "name": "type", - "value": "error30" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152725, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error11", - "last_checkpoint_time": 1633043263038, - "entity": [ - { - "name": "type", - "value": "error11" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152712, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error10", - "last_checkpoint_time": 1633043255533, - "entity": [ - { - "name": "type", - "value": "error10" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152719, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error7", - "last_checkpoint_time": 1633043258826, - "entity": [ - { - "name": "type", - "value": "error7" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152708, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error8", - "last_checkpoint_time": 1633043259841, - "entity": [ - { - "name": "type", - "value": "error8" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152721, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error12", - "last_checkpoint_time": 1633043261989, - "entity": [ - { - "name": "type", - "value": "error12" - } - ] - }, { "detector_id": "rlDtOHwBD5tpxlbyW7Nt", "model_type": "entity", @@ -2925,7 +2395,7 @@ POST _plugins/_alerting/monitors { "search": { "indices": [ - ".opendistro-anomaly-results*" + ".opensearch-anomaly-results*" ], "query": { "size": 1, @@ -3031,7 +2501,7 @@ POST _plugins/_alerting/monitors { "search": { "indices": [ - ".opendistro-anomaly-results*" + ".opensearch-anomaly-results*" ], "query": { "size": 1, @@ -3144,14 +2614,12 @@ GET _plugins/_anomaly_detection/detectors//_profile/, ```json GET _plugins/_anomaly_detection/detectors//_profile - { - "state":"DISABLED", - "error":"Stopped detector: AD models memory usage exceeds our limit." + "state": "DISABLED", + "error": "Stopped detector: AD models memory usage exceeds our limit." } GET _plugins/_anomaly_detection/detectors//_profile?_all=true&pretty - { "state": "RUNNING", "error": "", @@ -3256,246 +2724,253 @@ GET _plugins/_anomaly_detection/detectors//_profile?_all=true&pretty "node_id": "dIyavWhmSYWGz65b4u-lpQ" }, { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error1", + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error27", "entity": [ { "name": "type", - "value": "error1" - } - ], - "model_size_in_bytes": 403491, - "node_id": "dIyavWhmSYWGz65b4u-lpQ" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error22", - "entity": [ - { - "name": "type", - "value": "error22" - } - ], - "model_size_in_bytes": 403491, - "node_id": "dIyavWhmSYWGz65b4u-lpQ" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error32", - "entity": [ - { - "name": "type", - "value": "error32" - } - ], - "model_size_in_bytes": 403491, - "node_id": "dIyavWhmSYWGz65b4u-lpQ" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error23", - "entity": [ - { - "name": "type", - "value": "error23" - } - ], - "model_size_in_bytes": 403491, - "node_id": "dIyavWhmSYWGz65b4u-lpQ" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error26", - "entity": [ - { - "name": "type", - "value": "error26" - } - ], - "model_size_in_bytes": 403491, - "node_id": "dIyavWhmSYWGz65b4u-lpQ" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error14", - "entity": [ - { - "name": "type", - "value": "error14" - } - ], - "model_size_in_bytes": 403491, - "node_id": "dIyavWhmSYWGz65b4u-lpQ" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error9", - "entity": [ - { - "name": "type", - "value": "error9" - } - ], - "model_size_in_bytes": 403491, - "node_id": "dIyavWhmSYWGz65b4u-lpQ" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error19", - "entity": [ - { - "name": "type", - "value": "error19" - } - ], - "model_size_in_bytes": 403491, - "node_id": "dIyavWhmSYWGz65b4u-lpQ" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error18", - "entity": [ - { - "name": "type", - "value": "error18" - } - ], - "model_size_in_bytes": 403491, - "node_id": "dIyavWhmSYWGz65b4u-lpQ" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error29", - "entity": [ - { - "name": "type", - "value": "error29" - } - ], - "model_size_in_bytes": 403491, - "node_id": "dIyavWhmSYWGz65b4u-lpQ" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error5", - "entity": [ - { - "name": "type", - "value": "error5" + "value": "error27" } ], "model_size_in_bytes": 403491, "node_id": "2hEGbUw6ShaiKe05n_xLdA" + } + ], + "total_size_in_bytes": 12911712, + "init_progress": { + "percentage": "100%" + }, + "total_entities": 33, + "active_entities": 32, + "ad_task": { + "ad_task": { + "task_id": "Os4HOXwBCi2h__AONgpc", + "last_update_time": 1633044347855, + "started_by": "admin", + "state": "RUNNING", + "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "task_progress": 0, + "init_progress": 0, + "execution_start_time": 1633044346460, + "is_latest": true, + "task_type": "HISTORICAL_HC_DETECTOR", + "coordinating_node": "2hEGbUw6ShaiKe05n_xLdA", + "detector": { + "name": "test-detector", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" + ], + "filter_query": { + "match_all": { + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 5, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "7VDtOHwBD5tpxlbyWqPs", + "feature_name": "test-feature", + "feature_enabled": true, + "aggregation_query": { + "test_feature": { + "sum": { + "field": "value" + } + } + } + } + ], + "ui_metadata": { + "features": { + "test-feature": { + "aggregationBy": "sum", + "aggregationOf": "value", + "featureType": "simple_aggs" + } + }, + "filters": [] + }, + "last_update_time": 1633042652012, + "category_field": [ + "type" + ], + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": null + }, + "detector_type": "MULTI_ENTITY" + }, + "detection_date_range": { + "start_time": 1632437820000, + "end_time": 1633042620000 + }, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + } }, + "node_id": "2hEGbUw6ShaiKe05n_xLdA", + "task_id": "Os4HOXwBCi2h__AONgpc", + "task_type": "HISTORICAL_HC_DETECTOR", + "detector_task_slots": 10, + "total_entities_count": 32, + "pending_entities_count": 22, + "running_entities_count": 10, + "running_entities": [ + "error9", + "error8", + "error7", + "error6", + "error5", + "error4", + "error32", + "error31", + "error30", + "error3" + ], + "entity_task_profiles": [ + { + "shingle_size": 8, + "rcf_total_updates": 994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "bCtWtxWPThq0BIn5P5I4Xw", + "entity": [ + { + "name": "type", + "value": "error6" + } + ], + "task_id": "P84HOXwBCi2h__AOOgrC", + "task_type": "HISTORICAL_HC_ENTITY" + }, + { + "shingle_size": 8, + "rcf_total_updates": 994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "dIyavWhmSYWGz65b4u-lpQ", + "entity": [ + { + "name": "type", + "value": "error4" + } + ], + "task_id": "Kc4HOXwBCi2h__AOOw6Y", + "task_type": "HISTORICAL_HC_ENTITY" + }, + { + "shingle_size": 8, + "rcf_total_updates": 994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "dIyavWhmSYWGz65b4u-lpQ", + "entity": [ + { + "name": "type", + "value": "error8" + } + ], + "task_id": "Pc4HOXwBCi2h__AOOgqJ", + "task_type": "HISTORICAL_HC_ENTITY" + }, + { + "shingle_size": 8, + "rcf_total_updates": 994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "2hEGbUw6ShaiKe05n_xLdA", + "entity": [ + { + "name": "type", + "value": "error7" + } + ], + "task_id": "Ps4HOXwBCi2h__AOOgqh", + "task_type": "HISTORICAL_HC_ENTITY" + } + ] + }, + "model_count": 32 +} + +GET _plugins/_anomaly_detection/detectors//_profile/total_size_in_bytes +{ + "total_size_in_bytes": 13369344 +} +``` + +You can see the `ad_task` field only for a historical detector. + +The `model_count` parameter shows the total number of models that a detector runs in memory. This is useful if you have several models running on your cluster and want to know the count. + +If you configured the category field, you can see the number of unique values in the field and all active entities with models running in memory. + +You can use this data to estimate how much memory is required for anomaly detection so you can decide how to size your cluster. For example, if a detector has one million entities and only 10 of them are active in memory, you need to scale your cluster up or out. + +#### Request + +```json +GET _plugins/_anomaly_detection/detectors//_profile?_all=true&pretty +{ + "state": "RUNNING", + "error": "", + "models": [ { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error6", + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error20", "entity": [ { "name": "type", - "value": "error6" + "value": "error20" } ], "model_size_in_bytes": 403491, - "node_id": "2hEGbUw6ShaiKe05n_xLdA" + "node_id": "bCtWtxWPThq0BIn5P5I4Xw" }, { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error4", + "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error13", "entity": [ { "name": "type", - "value": "error4" + "value": "error13" } ], "model_size_in_bytes": 403491, - "node_id": "2hEGbUw6ShaiKe05n_xLdA" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error2", - "entity": [ - { - "name": "type", - "value": "error2" - } - ], - "model_size_in_bytes": 403491, - "node_id": "2hEGbUw6ShaiKe05n_xLdA" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error30", - "entity": [ - { - "name": "type", - "value": "error30" - } - ], - "model_size_in_bytes": 403491, - "node_id": "2hEGbUw6ShaiKe05n_xLdA" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error11", - "entity": [ - { - "name": "type", - "value": "error11" - } - ], - "model_size_in_bytes": 403491, - "node_id": "2hEGbUw6ShaiKe05n_xLdA" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error10", - "entity": [ - { - "name": "type", - "value": "error10" - } - ], - "model_size_in_bytes": 403491, - "node_id": "2hEGbUw6ShaiKe05n_xLdA" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error7", - "entity": [ - { - "name": "type", - "value": "error7" - } - ], - "model_size_in_bytes": 403491, - "node_id": "2hEGbUw6ShaiKe05n_xLdA" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error8", - "entity": [ - { - "name": "type", - "value": "error8" - } - ], - "model_size_in_bytes": 403491, - "node_id": "2hEGbUw6ShaiKe05n_xLdA" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error12", - "entity": [ - { - "name": "type", - "value": "error12" - } - ], - "model_size_in_bytes": 403491, - "node_id": "2hEGbUw6ShaiKe05n_xLdA" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error15", - "entity": [ - { - "name": "type", - "value": "error15" - } - ], - "model_size_in_bytes": 403491, - "node_id": "2hEGbUw6ShaiKe05n_xLdA" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error17", - "entity": [ - { - "name": "type", - "value": "error17" - } - ], - "model_size_in_bytes": 403491, - "node_id": "2hEGbUw6ShaiKe05n_xLdA" + "node_id": "bCtWtxWPThq0BIn5P5I4Xw" }, { "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error27", @@ -3681,70 +3156,6 @@ GET _plugins/_anomaly_detection/detectors//_profile?_all=true&pretty "task_id": "PM4HOXwBCi2h__AOOgp3", "task_type": "HISTORICAL_HC_ENTITY" }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "dIyavWhmSYWGz65b4u-lpQ", - "entity": [ - { - "name": "type", - "value": "error31" - } - ], - "task_id": "LM4HOXwBCi2h__AOOw7v", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "dIyavWhmSYWGz65b4u-lpQ", - "entity": [ - { - "name": "type", - "value": "error4" - } - ], - "task_id": "Kc4HOXwBCi2h__AOOw6Y", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "dIyavWhmSYWGz65b4u-lpQ", - "entity": [ - { - "name": "type", - "value": "error30" - } - ], - "task_id": "Lc4HOXwBCi2h__AOPA4R", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "dIyavWhmSYWGz65b4u-lpQ", - "entity": [ - { - "name": "type", - "value": "error8" - } - ], - "task_id": "Pc4HOXwBCi2h__AOOgqJ", - "task_type": "HISTORICAL_HC_ENTITY" - }, { "shingle_size": 8, "rcf_total_updates": 994, @@ -3797,70 +3208,125 @@ GET _plugins/_anomaly_detection/detectors//_profile?_all=true&pretty }, "model_count": 32 } - -GET _plugins/_anomaly_detection/detectors//_profile/total_size_in_bytes - -{ - "total_size_in_bytes" : 13369344 -} ``` -The `model_count` parameter shows the total number of models that a detector runs in memory. This is useful if you have several models running on your cluster and want to know the count. +For a single-entity detector: -If you configured the category field, you can see the number of unique values in the field and all active entities with models running in memory. - -You can use this data to estimate how much memory is required for anomaly detection so you can decide how to size your cluster. For example, if a detector has one million entities and only 10 of them are active in memory, you need to scale your cluster up or out. - -#### Request +#### Sample response ```json -GET _plugins/_anomaly_detection/detectors//_profile?_all=true&pretty - { - "state": "RUNNING", - "models": [ - { - "model_id": "T4c3dXUBj-2IZN7itix__entity_i-00f28ec1eb8997684", - "model_size_in_bytes": 712480, - "node_id": "g6pmr547QR-CfpEvO67M4g" - }, - { - "model_id": "T4c3dXUBj-2IZN7itix__entity_i-00f28ec1eb8997685", - "model_size_in_bytes": 712480, - "node_id": "g6pmr547QR-CfpEvO67M4g" - }, - { - "model_id": "T4c3dXUBj-2IZN7itix__entity_i-00f28ec1eb8997686", - "model_size_in_bytes": 712480, - "node_id": "g6pmr547QR-CfpEvO67M4g" - }, - { - "model_id": "T4c3dXUBj-2IZN7itix__entity_i-00f28ec1eb8997680", - "model_size_in_bytes": 712480, - "node_id": "g6pmr547QR-CfpEvO67M4g" - }, - { - "model_id": "T4c3dXUBj-2IZN7itix__entity_i-00f28ec1eb8997681", - "model_size_in_bytes": 712480, - "node_id": "g6pmr547QR-CfpEvO67M4g" - }, - { - "model_id": "T4c3dXUBj-2IZN7itix__entity_i-00f28ec1eb8997682", - "model_size_in_bytes": 712480, - "node_id": "g6pmr547QR-CfpEvO67M4g" - }, - { - "model_id": "T4c3dXUBj-2IZN7itix__entity_i-00f28ec1eb8997683", - "model_size_in_bytes": 712480, - "node_id": "g6pmr547QR-CfpEvO67M4g" - } - ], - "total_size_in_bytes": 4987360, + "state": "INIT", + "total_size_in_bytes": 0, "init_progress": { - "percentage": "100%" + "percentage": "0%", + "needed_shingles": 128 }, - "total_entities": 7, - "active_entities": 7 + "ad_task": { + "ad_task": { + "task_id": "cfUNOXwBFLNqSEcxAlde", + "last_update_time": 1633044731640, + "started_by": "admin", + "state": "RUNNING", + "detector_id": "qL4NOXwB__6eNorTAKtJ", + "task_progress": 0.49603173, + "init_progress": 1, + "current_piece": 1632739800000, + "execution_start_time": 1633044726365, + "is_latest": true, + "task_type": "HISTORICAL_SINGLE_ENTITY", + "coordinating_node": "bCtWtxWPThq0BIn5P5I4Xw", + "worker_node": "dIyavWhmSYWGz65b4u-lpQ", + "detector": { + "name": "detector1", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" + ], + "filter_query": { + "match_all": { + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 5, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "p74NOXwB__6eNorTAKss", + "feature_name": "test-feature", + "feature_enabled": true, + "aggregation_query": { + "test_feature": { + "sum": { + "field": "value" + } + } + } + } + ], + "ui_metadata": { + "features": { + "test-feature": { + "aggregationBy": "sum", + "aggregationOf": "value", + "featureType": "simple_aggs" + } + }, + "filters": [] + }, + "last_update_time": 1633044725832, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "SINGLE_ENTITY" + }, + "detection_date_range": { + "start_time": 1632439925885, + "end_time": 1633044725885 + }, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + } + }, + "shingle_size": 8, + "rcf_total_updates": 1994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "dIyavWhmSYWGz65b4u-lpQ", + "detector_task_slots": 1 + } } ``` @@ -3868,11 +3334,10 @@ The `total_entities` parameter shows you the total number of entities including Getting the total count of entities is an expensive operation for a detector with more than one category field. By default, a real-time detector counts the number of entities up to a value of 10,000 and a historical detector counts the number of entities up to a value of 1,000. -The `profile` operation also provides information about each entity, such as the entity’s `last_sample_timestamp` and `last_active_timestamp`. +The `profile` operation also provides information about each entity, such as the entity’s `last_sample_timestamp` and `last_active_timestamp`. `last_sample_timestamp` shows the last document in the input data source index containing the entity, while `last_active_timestamp` shows the timestamp when the entity’s model was last seen in the model cache. If there are no anomaly results for an entity, either the entity doesn't have any sample data or its model is removed from the model cache. - `last_sample_timestamp` shows the last document in the input data source index containing the entity, while `last_active_timestamp` shows the timestamp when the entity’s model was last seen in the model cache. #### Request diff --git a/_monitoring-plugins/ad/index.md b/_monitoring-plugins/ad/index.md index 87417e0f..b02f14a0 100644 --- a/_monitoring-plugins/ad/index.md +++ b/_monitoring-plugins/ad/index.md @@ -26,12 +26,11 @@ To first test with sample streaming data, you can try out one of the preconfigur A detector is an individual anomaly detection task. You can define multiple detectors, and all the detectors can run simultaneously, with each analyzing data from different sources. -1. Choose **Create Detector**. +1. Choose **Create detector**. 1. Enter a name and brief description. Make sure the name is unique and descriptive enough to help you to identify the purpose of the detector. 1. For **Data source**, choose the index you want to use as the data source. You can optionally use index patterns to choose multiple indices. -1. (Optional) For **Data filter**, filter the index you chose as the data source. From the **Data filter** menu, design your filter query by selecting **Field**, **Operator**, and **Value**, or choose **Use query DSL** and add your own JSON filter query. +1. (Optional) For **Data filter**, filter the index you chose as the data source. From the **Data filter** menu, choose **Add data filter**, and then design your filter query by selecting **Field**, **Operator**, and **Value**, or choose **Use query DSL** and add your own JSON filter query. 1. Select the **Timestamp field** in your index. -1. (Optional) For **Data filter**, filter the index you chose as the data source. From the **Filter type** menu, choose **Visual filter**, and then design your filter query by selecting **Fields**, **Operator**, and **Value**, or choose **Custom Expression** and add your own JSON filter query. 1. For **Operation settings**, define the **Detector interval**, which is the time interval at which the detector collects data. - The detector aggregates the data in this interval, then feeds the aggregated result into the anomaly detection model. The shorter you set this interval, the fewer data points the detector aggregates. @@ -46,7 +45,9 @@ Setting the window delay to 1 minute shifts the interval window to 1:49 - 1:59, After you define the detector, the next step is to configure the model. -### Step 2: Add features to your detector +### Step 2: Configure the model + +#### Add features to your detector A feature is the field in your index that you want to check for anomalies. A detector can discover anomalies across one or more features. You must choose an aggregation method for each feature: `average()`, `count()`, `sum()`, `min()`, or `max()`. The aggregation method determines what constitutes an anomaly. @@ -55,7 +56,7 @@ For example, if you choose `min()`, the detector focuses on finding anomalies ba A multi-feature model correlates anomalies across all its features. The [curse of dimensionality](https://en.wikipedia.org/wiki/Curse_of_dimensionality) makes it less likely for multi-feature models to identify smaller anomalies as compared to a single-feature model. Adding more features might negatively impact the [precision and recall](https://en.wikipedia.org/wiki/Precision_and_recall) of a model. A higher proportion of noise in your data might further amplify this negative impact. Selecting the optimal feature set is usually an iterative process. By default, the maximum number of features for a detector is 5. You can adjust this limit with the `plugins.anomaly_detection.max_anomaly_features` setting. {: .note } -1. On the **Configure Model** page, enter the **Feature name** and check **Enabled feature name**. +1. On the **Configure Model** page, enter the **Feature name** and check **Enable feature**. 1. For **Find anomalies based on**, choose the method to find anomalies. For **Field Value**, choose the **aggregation method**. Or choose **Custom expression**, and add your own JSON aggregation query. 1. Select a field. @@ -128,8 +129,8 @@ Analyze anomalies with the following visualizations: - **Live anomalies** - displays live anomaly results for the last 60 intervals. For example, if the interval is 10, it shows results for the last 600 minutes. The chart refreshes every 30 seconds. - **Anomaly history** (for historical analysis) / **Anomaly overview** (for real-time results) - plots the anomaly grade with the corresponding measure of confidence. -- **Feature breakdown** - plots the features based on the aggregation method. You can vary the date-time range of the detector. - **Anomaly occurrence** - shows the `Start time`, `End time`, `Data confidence`, and `Anomaly grade` for each detected anomaly. +- **Feature breakdown** - plots the features based on the aggregation method. You can vary the date-time range of the detector. `Anomaly grade` is a number between 0 and 1 that indicates how anomalous a data point is. An anomaly grade of 0 represents “not an anomaly,” and a non-zero value represents the relative severity of the anomaly. diff --git a/_monitoring-plugins/ad/settings.md b/_monitoring-plugins/ad/settings.md index 8d3cc5b1..eaebdee2 100644 --- a/_monitoring-plugins/ad/settings.md +++ b/_monitoring-plugins/ad/settings.md @@ -28,16 +28,21 @@ Setting | Default | Description `plugins.anomaly_detection.max_anomaly_detectors` | 1,000 | The maximum number of non-high cardinality detectors (no category field) users can create. `plugins.anomaly_detection.max_multi_entity_anomaly_detectors` | 10 | The maximum number of high cardinality detectors (with category field) in a cluster. `plugins.anomaly_detection.max_anomaly_features` | 5 | The maximum number of features for a detector. -`plugins.anomaly_detection.ad_result_history_rollover_period` | 12h | How often the rollover condition is checked. If `true`, the plugin rolls over the result index to a new index. -`plugins.anomaly_detection.ad_result_history_max_docs` | 250,000,000 | The maximum number of documents in one result index. The plugin only counts refreshed documents in the primary shards. +`plugins.anomaly_detection.ad_result_history_rollover_period` | 12h | How often the rollover condition is checked. If `true`, the anomaly detection plugin rolls over the result index to a new index. +`plugins.anomaly_detection.ad_result_history_max_docs` | 250,000,000 | The maximum number of documents in one result index. The anomaly detection plugin only counts refreshed documents in the primary shards. `plugins.anomaly_detection.ad_result_history_max_docs_per_shard` | 1,350,000,000 | The maximum number of documents in a single shard of the result index. The anomaly detection plugin only counts the refreshed documents in the primary shards. -`plugins.anomaly_detection.max_entities_per_query` | 1,000,000 | The maximum unique values per detection interval for high cardinality detectors. By default, if the category field has more than 1,000 unique values in a detector interval, the plugin selects the top 1,000 values and orders them by `doc_count`. -`plugins.anomaly_detection.max_entities_for_preview` | 5 | The maximum unique category field values displayed with the preview operation for high cardinality detectors. If the category field has more than 30 unique values, the plugin selects the top 30 values and orders them by `doc_count`. +`plugins.anomaly_detection.max_entities_per_query` | 1,000,000 | The maximum unique values per detection interval for high cardinality detectors. By default, if the category field(s) have more than the configured unique values in a detector interval, the anomaly detection plugin orders them by the natural ordering of categorical values (for example, entity `ab` comes before `bc`) and then selects the top values. +`plugins.anomaly_detection.max_entities_for_preview` | 5 | The maximum unique category field values displayed with the preview operation for high cardinality detectors. By default, if the category field(s) have more than the configured unique values in a detector interval, the anomaly detection plugin orders them by the natural ordering of categorical values (for example, entity `ab` comes before `bc`) and then selects the top values. `plugins.anomaly_detection.max_primary_shards` | 10 | The maximum number of primary shards an anomaly detection index can have. -`plugins.anomaly_detection.filter_by_backend_roles` | False | When you enable the security plugin and set this to `true`, the plugin filters results based on the user's backend role(s). -`plugins.anomaly_detection.max_batch_task_per_node` | 10 | Starting a historical detector triggers a batch task. This setting is the number of batch tasks that you can run per data node. You can tune this setting from 1 to 1000. If the data nodes can't support all batch tasks and you're not sure if the data nodes are capable of running more historical detectors, add more data nodes instead of changing this setting to a higher value. +`plugins.anomaly_detection.filter_by_backend_roles` | False | When you enable the security plugin and set this to `true`, the anomaly detection plugin filters results based on the user's backend role(s). +`plugins.anomaly_detection.max_batch_task_per_node` | 10 | Starting a historical detector triggers a batch task. This setting is the number of batch tasks that you can run per data node. You can tune this setting from 1 to 1,000. If the data nodes can't support all batch tasks and you're not sure if the data nodes are capable of running more historical detectors, add more data nodes instead of changing this setting to a higher value. `plugins.anomaly_detection.max_old_ad_task_docs_per_detector` | 1 | You can run the same historical detector many times. For each run, the anomaly detection plugin creates a new task. This setting is the number of previous tasks the plugin keeps. Set this value to at least 1 to track its last run. You can keep a maximum of 1,000 old tasks to avoid overwhelming the cluster. -`plugins.anomaly_detection.batch_task_piece_size` | 1,000 | The date range for a historical task is split into smaller pieces and the anomaly detection plugin runs the task piece by piece. Each piece contains 1,000 detection intervals by default. For example, if detector interval is 1 minute and one piece is 1000 minutes, the feature data is queried every 1,000 minutes. You can change this setting from 1 to 10,000. +`plugins.anomaly_detection.batch_task_piece_size` | 1,000 | The date range for a historical task is split into smaller pieces and the anomaly detection plugin runs the task piece by piece. Each piece contains 1,000 detection intervals by default. For example, if detector interval is 1 minute and one piece is 1,000 minutes, the feature data is queried every 1,000 minutes. You can change this setting from 1 to 10,000. `plugins.anomaly_detection.batch_task_piece_interval_seconds` | 5 | Add a time interval between historical detector tasks. This interval prevents the task from consuming too much of the available resources and starving other operations like search and bulk index. You can change this setting from 1 to 600 seconds. -`plugins.anomaly_detection.max_top_entities_for_historical_analysis` | 1,000 | The maximum number of top entities that you run for a high-cardinality detector historical analysis. -`plugins.anomaly_detection.max_running_entities_per_detector_for_historical_analysis` | 10 | How many entity tasks you can run in parallel for one HC detector. The cluster availble task slots will impact how many entities can run in parallel as well. For example, the cluster has 3 data nodes, each data node has 10 task slots by default. But if we have already started 2 HC detectors and each HC running 10 entities, and start a single-flow detector which takes 1 task slot, then the availabe task slots will be 10 * 3 - 10 * 2 - 1 = 9. Then, if we start a new HC detector, it can only run 9 entities in parallel, not 10. +`plugins.anomaly_detection.max_top_entities_for_historical_analysis` | 1,000 | The maximum number of top entities that you run for a high cardinality detector historical analysis. +`plugins.anomaly_detection.max_running_entities_per_detector_for_historical_analysis` | 10 | The number of entity tasks that you can run in parallel for a single high cardinality detector. The task slots available on your cluster also impact how many entities run in parallel. If a cluster has 3 data nodes, each data node has 10 task slots by default. Say you already have two high cardinality detectors and each of them run 10 entities. If you start a single-flow detector that takes 1 task slot, the number of task slots available is 10 * 3 - 10 * 2 - 1 = 9. if you now start a new high cardinality detector, the detector can only run 9 entities in parallel and not 10. +`plugins.anomaly_detection.max_cached_deleted_tasks` | 1,000 | You can rerun historical analysis for a single detector as many times as you like. The anomaly detection plugin only keeps a limited number of old tasks, by default 1 old task. If you run historical analysis three times for a detector, the oldest task is deleted. Because historical analysis generates a number of anomaly results in a short span of time, it's necessary to clean up anomaly results for a deleted task. With this field, you can configure how many deleted tasks you can cache at most. The plugin cleans up a task's results when it's deleted. If the plugin fails to do this cleanup, it adds the task's results into a cache and an hourly cron job performs the cleanup. After an hour, if still you find an old task result in the cache, use the [delete detector results API]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/api/#delete-detector-results) to delete the task result manually. +`plugins.anomaly_detection.delete_anomaly_result_when_delete_detector` | False | Whether the anomaly detection plugin deletes the anomaly result when you delete a detector. If you want to save some disk space, especially if you've high cardinality detectors generating a lot of results, set this field to true. Alternatively, you can use the [delete detector results API]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/api/#delete-detector-results) to manually delete the results. +`plugins.anomaly_detection.dedicated_cache_size` | 10 | If the real-time analysis of a high cardinality detector starts successfully, the anomaly detection plugin guarantees keeping 10 (dynamically adjustable via this setting) entities' models in memory per node. If the number of entities exceeds this limit, the plugin puts the extra entities' models in a memory space shared by all detectors. The actual number of entities varies based on the memory that you've available and the frequencies of the entities. If you'd like the plugin to guarantee keeping more entities' models in memory and if you're cluster has sufficient memory, you can increase this setting value. +`plugins.anomaly_detection.max_concurrent_preview` | 2 | The maximum number of concurrent previews. You can use this setting to limit resource usage. +`plugins.anomaly_detection.model_max_size_percent` | 0.1 | The upper bound of the memory percentage for a model. From f12f4efc02cff58961ddf627878e7defdbe11cca Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Mon, 4 Oct 2021 12:13:31 -0700 Subject: [PATCH 060/229] minor fix --- _monitoring-plugins/ad/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_monitoring-plugins/ad/api.md b/_monitoring-plugins/ad/api.md index e3db5aac..ab12a8f8 100644 --- a/_monitoring-plugins/ad/api.md +++ b/_monitoring-plugins/ad/api.md @@ -265,7 +265,7 @@ Passes a date range to the anomaly detector to return any anomalies within that #### Request ```json -POST _plugins/_anomaly_detection/detectors/_preview +POST _plugins/_anomaly_detection/detectors//_preview { "period_start": 1612982516000, "period_end": 1614278539000, From f0f770bfad04f4792c5c264b2957c28d14677865 Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Mon, 4 Oct 2021 12:32:08 -0700 Subject: [PATCH 061/229] removed validate API --- _monitoring-plugins/ad/api.md | 139 ---------------------------------- 1 file changed, 139 deletions(-) diff --git a/_monitoring-plugins/ad/api.md b/_monitoring-plugins/ad/api.md index ab12a8f8..f0d8e4c5 100644 --- a/_monitoring-plugins/ad/api.md +++ b/_monitoring-plugins/ad/api.md @@ -1230,145 +1230,6 @@ DELETE _plugins/_anomaly_detection/detectors/results --- -## Validate detector -Introduced 1.1 -{: .label .label-purple } - -Validates detector before creating. Lists any invalid fields in your configuration and recommends ways to fix these invalid fields. - -#### Request - -```json -POST _plugins/_anomaly_detection/detectors/_validate/detector,model -{ - "name": "test-detector", - "description": "Test detector", - "time_field": "timestamp", - "indices": [ - "order*" - ], - "feature_attributes": [ - { - "feature_name": "total_order", - "feature_enabled": true, - "aggregation_query": { - "total_order": { - "sum": { - "field": "value" - } - } - } - } - ], - "filter_query": { - "bool": { - "filter": [ - { - "exists": { - "field": "value", - "boost": 1 - } - } - ], - "adjust_pure_negative": true, - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "category_field": [ - "hc_field" - ], - "shingle_size": 8 -} -``` - - -#### Sample response - -```json -{ - "detector": { - "name": { - "message": "name should be set|duplicate" - }, - "time_field": { - "message": "time_field should be set missing|not_exist" - }, - "indices": { - "message": "should be set|not_exist|empty" - }, - "feature_attributes": { - "problematic_feature_name1": { - "message": "{field} is invalid with {exceptionMessage}" - } - "problematic_feature_name2": { - "message": "{field} is invalid with {exceptionMessage}" - }, - "message": "there exists non-numeric field|duplicate feature names|over 5 features|duplicate feature aggregation query names" - }, - "detection_interval": { - "message": "detection_interval should be set|Interval should be non-negative|unit is not supported" - }, - "category_field": { - "message": "must only 1 field, and must be IP address or keyword type" - }, - "shingle_size": { - "message": "must be between 1 and 1000" - }, - }, - "model": { - "filter_query": { - "message": "data is too sparse after filter_query is applied" - }, - "detection_interval": { - "suggested_value": { - "period": { - "interval": 1, - "unit": "Minutes" - } - } - "message": "use suggested value|no suggested value found, ingest more data" - }, - "category_field": { - "message": "data with {category_field} is too sparse, ingest more data" - }, - "feature_attributes": { - "problematic_feature_name1": { - "message": "data is too sparse, ingest more data with this {field}" - } - "problematic_feature_name2": { - "message": "data is too sparse, ingest more data with this {field}" - }, - "message": "data is too sparse, ingest more data" - }, - "memory": { - "message": "model size exceeds memory limit, please stop/delete unused detectors, or reduce shingle size or number of features" - }, - "window_delay": { - "suggested_value": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "message": "use suggested value(if it exists), and ingest more data if possible" - } - } -} -``` - ---- ## Update detector Introduced 1.0 From 72c710d5a3a80818e1aee0edf9f1dfa632af7906 Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Mon, 4 Oct 2021 12:38:02 -0700 Subject: [PATCH 062/229] historical detector to historical analysis --- _monitoring-plugins/ad/api.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/_monitoring-plugins/ad/api.md b/_monitoring-plugins/ad/api.md index f0d8e4c5..c6bd39e6 100644 --- a/_monitoring-plugins/ad/api.md +++ b/_monitoring-plugins/ad/api.md @@ -2024,13 +2024,15 @@ GET _plugins/_anomaly_detection/stats/ ``` The `model_count` parameter shows the total number of models running on each node’s memory. -Historical detectors contain the following additional fields: +For historical analysis, you see the values for the following fields: - `ad_total_batch_task_execution_count` - `ad_executing_batch_task_count` - `ad_canceled_batch_task_count` - `ad_batch_task_failure_count` +For real-time analysis, these values are 0. + #### Sample response ```json @@ -2795,7 +2797,7 @@ GET _plugins/_anomaly_detection/detectors//_profile/total_size_in_by } ``` -You can see the `ad_task` field only for a historical detector. +You can see the `ad_task` field only for historical analysis. The `model_count` parameter shows the total number of models that a detector runs in memory. This is useful if you have several models running on your cluster and want to know the count. From 1c10355a795d206bc8918e9fbfdb5816c6f99a11 Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Mon, 4 Oct 2021 12:41:37 -0700 Subject: [PATCH 063/229] minor fixes --- _monitoring-plugins/ad/api.md | 8 ++++---- _monitoring-plugins/ad/index.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/_monitoring-plugins/ad/api.md b/_monitoring-plugins/ad/api.md index c6bd39e6..d434ec7d 100644 --- a/_monitoring-plugins/ad/api.md +++ b/_monitoring-plugins/ad/api.md @@ -793,7 +793,7 @@ POST _plugins/_anomaly_detection/detectors/results/_search } ``` -You can query the anomaly results of a historical detector with the `task_id`: +You can query the anomaly results for historical analysis with the `task_id`: #### Request @@ -1570,7 +1570,7 @@ GET _plugins/_anomaly_detection/detectors/?job=true } ``` -Use `task=true` to get historical detector task information. +Use `task=true` to get historical analysis task information. #### Request @@ -3195,7 +3195,7 @@ For a single-entity detector: The `total_entities` parameter shows you the total number of entities including the number of category fields for a detector. -Getting the total count of entities is an expensive operation for a detector with more than one category field. By default, a real-time detector counts the number of entities up to a value of 10,000 and a historical detector counts the number of entities up to a value of 1,000. +Getting the total count of entities is an expensive operation for a detector with more than one category field. By default, a real-time detector counts the number of entities up to a value of 10,000 and historical analysis counts the number of entities up to a value of 1,000. The `profile` operation also provides information about each entity, such as the entity’s `last_sample_timestamp` and `last_active_timestamp`. `last_sample_timestamp` shows the last document in the input data source index containing the entity, while `last_active_timestamp` shows the timestamp when the entity’s model was last seen in the model cache. @@ -3237,7 +3237,7 @@ GET _plugins/_anomaly_detection/detectors//_profile?_all=true } ``` -For a historical detector, specify `_all` or `ad_task` to see information about its latest task: +For historical analysis, specify `_all` or `ad_task` to see information about its latest task: #### Request diff --git a/_monitoring-plugins/ad/index.md b/_monitoring-plugins/ad/index.md index b02f14a0..d41f6455 100644 --- a/_monitoring-plugins/ad/index.md +++ b/_monitoring-plugins/ad/index.md @@ -153,7 +153,7 @@ If you stop or delete a detector, make sure to delete any monitors associated wi To see all the configuration settings for a detector, choose the **Detector configuration** tab. 1. To make any changes to the detector configuration, or fine tune the time interval to minimize any false positives, go to the **Detector configuration** section and choose **Edit**. -- You need to stop a real-time or historical detector to change its configuration. Confirm that you want to stop the detector and proceed. +- You need to stop real-time or historical analysis to change its configuration. Confirm that you want to stop the detector and proceed. 1. To enable or disable features, in the **Features** section, choose **Edit** and adjust the feature settings as needed. After you make your changes, choose **Save and start detector**. ### Step 8: Manage your detectors From 16e8e1bbc47021588b6453791c047cbc3f109287 Mon Sep 17 00:00:00 2001 From: Liz Snyder Date: Mon, 4 Oct 2021 13:47:22 -0700 Subject: [PATCH 064/229] Clarify connection alias --- _replication-plugin/api.md | 17 +++++++++-------- _replication-plugin/auto-follow.md | 4 ++-- _replication-plugin/get-started.md | 16 +++++++++------- _replication-plugin/permissions.md | 6 ++---- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/_replication-plugin/api.md b/_replication-plugin/api.md index 792aaecf..58457c84 100644 --- a/_replication-plugin/api.md +++ b/_replication-plugin/api.md @@ -24,7 +24,7 @@ Initiate replication of an index from the leader cluster to the follower cluster ```json PUT /_plugins/_replication//_start { - "leader_alias":"", + "leader_alias":"", "leader_index":"", "use_roles":{ "leader_cluster_role":"", @@ -37,7 +37,7 @@ Specify the following options: Options | Description | Type | Required :--- | :--- |:--- |:--- | -`leader_alias` | The name of the leader cluster. This alias is the same as the remote cluster name used to set up a cross-cluster connection. | `string` | Yes +`leader_alias` | The name of the cross-cluster connection. You define this alias when you [set up a cross-cluster connection]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/#set-up-a-cross-cluster-connection). | `string` | Yes `leader_index` | The index on the leader cluster that you want to replicate. | `string` | Yes `use_roles` | The roles to use for all subsequent backend replication tasks between the indices. Specify a `leader_cluster_role` and `follower_cluster_role`. See [Map the leader and follower cluster roles]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). | `string` | If security plugin is enabled @@ -130,7 +130,7 @@ GET /_plugins/_replication//_status { "status" : "SYNCING", "reason" : "User initiated", - "leader_alias" : "leader-cluster", + "leader_alias" : "my-connection-name", "leader_index" : "leader-01", "follower_index" : "follower-01", "syncing_details" : { @@ -140,9 +140,10 @@ GET /_plugins/_replication//_status } } ``` - To include shard replication details in the response, add the `&verbose=true` parameter. +The leader and follower checkpoint values begin as negative integers and reflect the number of shards you have (-1 for one shard, -5 for five shards, and so on). The values increment to positive integers with each change that you make. For example, when you make a change on the leader index, the `leader_checkpoint` becomes `0`. The `follower_checkpoint` is initially still `-1` until the follower index pulls the change from the leader, at which point it increments to `0`. If the values are the same, it means the indices are fully synced. + ## Update settings Introduced 1.1 {: .label .label-purple } @@ -185,7 +186,7 @@ Make sure to note the names of all auto-follow patterns after you create them. T ```json POST /_plugins/_replication/_autofollow { - "leader_alias" : "", + "leader_alias" : "", "name": "", "pattern": "", "use_roles":{ @@ -199,7 +200,7 @@ Specify the following options: Options | Description | Type | Required :--- | :--- |:--- |:--- | -`leader_alias` | The name of the remote cluster to associate the pattern with. | `string` | Yes +`leader_alias` | The name of the cross-cluster connection. You define this alias when you [set up a cross-cluster connection]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/#set-up-a-cross-cluster-connection). | `string` | Yes `name` | A name for the auto-follow pattern. | `string` | Yes `pattern` | An array of index patterns to match against indices in the specified leader cluster. Supports wildcard characters. For example, `leader-*`. | `string` | Yes `use_roles` | The roles to use for all subsequent backend replication tasks between the indices. Specify a `leader_cluster_role` and `follower_cluster_role`. See [Map the leader and follower cluster roles]({{site.url}}{{site.baseurl}}/replication-plugin/permissions/#map-the-leader-and-follower-cluster-roles). | `string` | If security plugin is enabled @@ -225,7 +226,7 @@ Send this request to the follower cluster. ```json DELETE /_plugins/_replication/_autofollow { - "leader_alias" : "", + "leader_alias" : "", "name": "", } ``` @@ -234,7 +235,7 @@ Specify the following options: Options | Description | Type | Required :--- | :--- |:--- |:--- | -`leader_alias` | The name of the remote cluster that the pattern is associated with. | `string` | Yes +`leader_alias` | The name of the cross-cluster connection. You define this alias when you [set up a cross-cluster connection]({{site.url}}{{site.baseurl}}/replication-plugin/get-started/#set-up-a-cross-cluster-connection). | `string` | Yes `name` | The name of the pattern. | `string` | Yes #### Sample response diff --git a/_replication-plugin/auto-follow.md b/_replication-plugin/auto-follow.md index 2e06761e..4af6c61f 100644 --- a/_replication-plugin/auto-follow.md +++ b/_replication-plugin/auto-follow.md @@ -32,7 +32,7 @@ Create a replication rule on the follower cluster: ```bash curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/_autofollow?pretty' -d ' { - "leader_alias" : "leader-cluster", + "leader_alias" : "my-connection-alias", "name": "my-replication-rule", "pattern": "movies*", "use_roles":{ @@ -69,7 +69,7 @@ When you delete a replication rule, OpenSearch stops replicating *new* indices t ```bash curl -XDELETE -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/_autofollow?pretty' -d ' { - "leader_alias" : "leader-cluster", + "leader_alias" : "my-conection-alias", "name": "my-replication-rule" }' ``` diff --git a/_replication-plugin/get-started.md b/_replication-plugin/get-started.md index 30955b8c..e6b3c36a 100644 --- a/_replication-plugin/get-started.md +++ b/_replication-plugin/get-started.md @@ -132,7 +132,7 @@ docker inspect --format='{% raw %}{{range .NetworkSettings.Networks}}{{.IPAddres Cross-cluster replication follows a "pull" model, so most changes occur on the follower cluster, not the leader cluster. -On the follower cluster, add the leader cluster name and the IP address (with port 9300) for each seed node. Because this is a single-node cluster, you only have one seed node: +On the follower cluster, add the IP address (with port 9300) for each seed node. Because this is a single-node cluster, you only have one seed node. Provide a descriptive name for the connection, which you'll use in the request to start replication: ```bash curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_cluster/settings?pretty' -d ' @@ -140,7 +140,7 @@ curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://loca "persistent": { "cluster": { "remote": { - "leader-cluster": { + "my-connection-alias": { "seeds": ["172.22.0.3:9300"] } } @@ -157,12 +157,12 @@ To get started, create an index called `leader-01` on the leader cluster: curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/leader-01?pretty' ``` -Then start replication of that index from the follower cluster. In the request body, provide the leader cluster and index, along with the security roles that you want to use: +Then start replication from the follower cluster. In the request body, provide the connection name and leader index that you want to replicate, along with the security roles you want to use: ```bash curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_start?pretty' -d ' { - "leader_alias": "leader-cluster", + "leader_alias": "my-connection-alias", "leader_index": "leader-01", "use_roles":{ "leader_cluster_role": "all_access", @@ -186,7 +186,7 @@ curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/fol { "status" : "SYNCING", "reason" : "User initiated", - "leader_alias" : "leader-cluster", + "leader_alias" : "my-connection-alias", "leader_index" : "leader-01", "follower_index" : "follower-01", "syncing_details" : { @@ -197,7 +197,9 @@ curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/fol } ``` -Possible statuses are `SYNCING`, `BOOTSTRAPING`, `PAUSED`, and `REPLICATION NOT IN PROGRESS`. The leader and follower checkpoint values increment with each change and illustrate how many updates the follower is behind the leader. If the indices are fully synced, the values are the same. +Possible statuses are `SYNCING`, `BOOTSTRAPING`, `PAUSED`, and `REPLICATION NOT IN PROGRESS`. + +The leader and follower checkpoint values begin as negative numbers and reflect the number of shards you have (-1 for one shard, -5 for five shards, and so on). The values increment with each change and illustrate how many updates the follower is behind the leader. If the indices are fully synced, the values are the same. To confirm that replication is actually happening, add a document to the leader index: @@ -240,7 +242,7 @@ curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/fol { "status" : "PAUSED", "reason" : "User initiated", - "leader_alias" : "leader-cluster", + "leader_alias" : "my-connection-alias", "leader_index" : "leader-01", "follower_index" : "follower-01" } diff --git a/_replication-plugin/permissions.md b/_replication-plugin/permissions.md index 2204dba5..e4b3152c 100644 --- a/_replication-plugin/permissions.md +++ b/_replication-plugin/permissions.md @@ -18,7 +18,7 @@ Enable node-to-node encryption on both the leader and the follower cluster to en ## Basic permissions -In order for non-admin users to perform replication activities, they be mapped to the appropriate permissions. +In order for non-admin users to perform replication activities, they must be mapped to the appropriate permissions. The security plugin has two built-in roles that cover most replication use cases: `cross_cluster_replication_leader_full_access`, which provides replication permissions on the leader cluster, and `cross_cluster_replication_follower_full_access`, which provides replication permissions on the follower cluster. For descriptions of each, see [Predefined roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/users-roles#predefined-roles). @@ -26,9 +26,7 @@ If you don't want to use the default roles, you can combine individual replicati ## Map the leader and follower cluster roles -associates roles passed in the request to these replication jobs to run in the background - -The [start replication]({{site.url}}{{site.baseurl}}/replication-plugin/api/#start-replication) and [create replication rule]({{site.url}}{{site.baseurl}}/replication-plugin/api/#create-replication-rule) operations are special cases. They involve background processes on the leader and follower clusters that must be associated with roles. When you perform one of these actions, you must explicitly pass the `leader_cluster_role` and +The [start replication]({{site.url}}{{site.baseurl}}/replication-plugin/api/#start-replication) and [create replication rule]({{site.url}}{{site.baseurl}}/replication-plugin/api/#create-replication-rule) operations are special cases. They involve background processes on the leader and follower clusters that must be associated with roles. When you perform one of these actions, you must explicitly pass the `leader_cluster_role` and `follower_cluster_role` in the request, which OpenSearch then uses in all backend replication tasks. To enable non-admins to start replication and create replication rules, create an identical user on each cluster (for example, `replication_user`) and map them to the `cross_cluster_replication_leader_full_access` role on the remote cluster and `cross_cluster_replication_follower_full_access` on the follower cluster. For instructions, see [Map users to roles]({{site.url}}{{site.baseurl}}/security-plugin/access-control/users-roles/#map-users-to-roles). From 4b23a1a36ec8210efbfbe77d526b7bf383ff31d7 Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Mon, 4 Oct 2021 14:11:45 -0700 Subject: [PATCH 065/229] incorporated feedback --- _monitoring-plugins/ad/api.md | 1061 ++++++++++++++-------------- _monitoring-plugins/ad/settings.md | 12 +- 2 files changed, 532 insertions(+), 541 deletions(-) diff --git a/_monitoring-plugins/ad/api.md b/_monitoring-plugins/ad/api.md index d434ec7d..b6aaefe5 100644 --- a/_monitoring-plugins/ad/api.md +++ b/_monitoring-plugins/ad/api.md @@ -256,6 +256,530 @@ Options | Description | Type | Required --- +## Get detector +Introduced 1.0 +{: .label .label-purple } + +Returns all information about a detector based on the `detector_id`. + +#### Request + +```json +GET _plugins/_anomaly_detection/detectors/ +``` + +#### Sample response + +```json +{ + "_id" : "m4ccEnIBTXsGi3mvMt9p", + "_version" : 1, + "_primary_term" : 1, + "_seq_no" : 3, + "anomaly_detector" : { + "name" : "test-detector", + "description" : "Test detector", + "time_field" : "timestamp", + "indices" : [ + "order*" + ], + "filter_query" : { + "bool" : { + "filter" : [ + { + "exists" : { + "field" : "value", + "boost" : 1.0 + } + } + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + }, + "detection_interval" : { + "period" : { + "interval" : 1, + "unit" : "Minutes" + } + }, + "window_delay" : { + "period" : { + "interval" : 1, + "unit" : "Minutes" + } + }, + "schema_version" : 0, + "feature_attributes" : [ + { + "feature_id" : "mYccEnIBTXsGi3mvMd8_", + "feature_name" : "total_order", + "feature_enabled" : true, + "aggregation_query" : { + "total_order" : { + "sum" : { + "field" : "value" + } + } + } + } + ], + "last_update_time" : 1589441737319 + } +} +``` + + +Use `job=true` to get anomaly detection job information. + +#### Request + +```json +GET _plugins/_anomaly_detection/detectors/?job=true +``` + +#### Sample response + +```json +{ + "_id": "LJxGsXcBoDQA8W1Q--A1", + "_version": 1, + "_primary_term": 1, + "_seq_no": 0, + "anomaly_detector": { + "name": "test2", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" + ], + "filter_query": { + "match_all": { + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "K5xGsXcBoDQA8W1Q-uCF", + "feature_name": "F1", + "feature_enabled": "true", + "aggregation_query": { + "f_1": { + "sum": { + "field": "value" + } + } + } + } + ], + "last_update_time": 1613586955060, + "detector_type": "MULTI_ENTITY" + }, + "anomaly_detector_job": { + "name": "LJxGsXcBoDQA8W1Q--A1", + "schedule": { + "interval": { + "start_time": 1613587220387, + "period": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "enabled": "false", + "enabled_time": 1613587220387, + "last_update_time": 1613587289169, + "lock_duration_seconds": 60, + "disabled_time": 1613587289169 + }, + "anomaly_detection_task": { + "task_id": "WZ5LsXcBoDQA8W1QmUa3", + "last_update_time": 1613587349022, + "error": "Task cancelled by user", + "state": "STOPPED", + "detector_id": "LJxGsXcBoDQA8W1Q--A1", + "task_progress": 0.26321793, + "init_progress": 1, + "current_piece": 1611030900000, + "execution_start_time": 1613587257783, + "execution_end_time": 1613587349022, + "is_latest": "true", + "task_type": "HISTORICAL", + "coordinating_node": "NSw5j-3YQeGkH8KESVKlzw", + "worker_node": "NSw5j-3YQeGkH8KESVKlzw", + "detector": { + "name": "test2", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" + ], + "filter_query": { + "match_all": { + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "K5xGsXcBoDQA8W1Q-uCF", + "feature_name": "F1", + "feature_enabled": "true", + "aggregation_query": { + "f_1": { + "sum": { + "field": "value" + } + } + } + } + ], + "last_update_time": 1613586955060, + "detector_type": "MULTI_ENTITY" + } + } +} +``` + +Use `task=true` to get historical analysis task information. + +#### Request + +```json +GET _plugins/_anomaly_detection/detectors/?task=true +``` + +#### Sample response + +```json +{ + "_id": "BwzKQXcB89DLS7G9rg7Y", + "_version": 1, + "_primary_term": 2, + "_seq_no": 10, + "anomaly_detector": { + "name": "test-ylwu1", + "description": "test", + "time_field": "timestamp", + "indices": [ + "ser*" + ], + "filter_query": { + "match_all": { + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 10, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "BgzKQXcB89DLS7G9rg7G", + "feature_name": "F1", + "feature_enabled": true, + "aggregation_query": { + "f_1": { + "sum": { + "field": "value" + } + } + } + } + ], + "ui_metadata": { + "features": { + "F1": { + "aggregationBy": "sum", + "aggregationOf": "value", + "featureType": "simple_aggs" + } + } + }, + "last_update_time": 1611716538071, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "all_access", + "own_index" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "HISTORICAL_SINGLE_ENTITY", + "detection_date_range": { + "start_time": 1580094137997, + "end_time": 1611716537997 + } + }, + "anomaly_detection_task": { + "task_id": "sgxaRXcB89DLS7G9RfIO", + "last_update_time": 1611776648699, + "started_by": "admin", + "state": "FINISHED", + "detector_id": "BwzKQXcB89DLS7G9rg7Y", + "task_progress": 1, + "init_progress": 1, + "current_piece": 1611716400000, + "execution_start_time": 1611776279822, + "execution_end_time": 1611776648679, + "is_latest": true, + "task_type": "HISTORICAL", + "coordinating_node": "gs213KqjS4q7H4Bmn_ZuLA", + "worker_node": "PgfR3JhbT7yJMx7bwQ6E3w", + "detector": { + "name": "test-ylwu1", + "description": "test", + "time_field": "timestamp", + "indices": [ + "ser*" + ], + "filter_query": { + "match_all": { + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 10, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "BgzKQXcB89DLS7G9rg7G", + "feature_name": "F1", + "feature_enabled": true, + "aggregation_query": { + "f_1": { + "sum": { + "field": "value" + } + } + } + } + ], + "ui_metadata": { + "features": { + "F1": { + "aggregationBy": "sum", + "aggregationOf": "value", + "featureType": "simple_aggs" + } + } + }, + "last_update_time": 1611716538071, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "all_access", + "own_index" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "HISTORICAL_SINGLE_ENTITY", + "detection_date_range": { + "start_time": 1580094137997, + "end_time": 1611716537997 + } + }, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "all_access", + "own_index" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + } + } +} +``` + +--- + +## Update detector +Introduced 1.0 +{: .label .label-purple } + +Updates a detector with any changes, including the description or adding or removing of features. +To update a detector, you need to first stop the detector. + +#### Request + +```json +PUT _plugins/_anomaly_detection/detectors/ +{ + "name": "test-detector", + "description": "Test detector", + "time_field": "timestamp", + "indices": [ + "order*" + ], + "feature_attributes": [ + { + "feature_name": "total_order", + "feature_enabled": true, + "aggregation_query": { + "total_order": { + "sum": { + "field": "value" + } + } + } + } + ], + "filter_query": { + "bool": { + "filter": [ + { + "exists": { + "field": "value", + "boost": 1 + } + } + ], + "adjust_pure_negative": true, + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 10, + "unit": "MINUTES" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "MINUTES" + } + } +} +``` + + +#### Sample response + +```json +{ + "_id" : "m4ccEnIBTXsGi3mvMt9p", + "_version" : 2, + "_seq_no" : 4, + "_primary_term" : 1, + "anomaly_detector" : { + "name" : "test-detector", + "description" : "Test detector", + "time_field" : "timestamp", + "indices" : [ + "order*" + ], + "filter_query" : { + "bool" : { + "filter" : [ + { + "exists" : { + "field" : "value", + "boost" : 1.0 + } + } + ], + "adjust_pure_negative" : true, + "boost" : 1.0 + } + }, + "detection_interval" : { + "period" : { + "interval" : 10, + "unit" : "Minutes" + } + }, + "window_delay" : { + "period" : { + "interval" : 1, + "unit" : "Minutes" + } + }, + "schema_version" : 0, + "feature_attributes" : [ + { + "feature_id" : "xxokEnIBcpeWMD987A1X", + "feature_name" : "total_order", + "feature_enabled" : true, + "aggregation_query" : { + "total_order" : { + "sum" : { + "field" : "value" + } + } + } + } + ] + } +} +``` + +--- + ## Preview detector Introduced 1.0 {: .label .label-purple } @@ -937,15 +1461,6 @@ GET _plugins/_anomaly_detection/detectors/results/_search } ``` -You can specify the following options. - -Options | Description | Type | Required -:--- | :--- |:--- |:--- | -`anomalyThreshold` | Specify a threshold to filter out low anomaly grade results. Default is -1. Because the lowest possible anomaly grade is 0, -1 means that the detector returns all results. | `float` | No -`dateRangeFilter` | Specify the date range in:
- `startTime` (int): Start time to collect results. Add in milliseconds since the Unix Epoch.
- `endTime` (int): End time to collect results. Add in milliseconds since the Unix Epoch.
- `fieldName` (string): The field that you want to match the start and end time. | `object` | Yes -`entity` | Specify the entity name and value. Default is empty.
- `name` (string): Field name that you want to search in.
- `value` (string): Entity value that you want to search for. | `object` | No -`sort` | Sort the result by a field in a certain order. Default is empty. Properties of `sort`:
- `direction` (string): Specify "desc" or "asc" for descending or ascending order.
- `field` (string): Order the results by a field. | `object` | No - --- ## Search detector tasks @@ -1228,530 +1743,6 @@ DELETE _plugins/_anomaly_detection/detectors/results } ``` ---- - - -## Update detector -Introduced 1.0 -{: .label .label-purple } - -Updates a detector with any changes, including the description or adding or removing of features. -To update a detector, you need to first stop the detector. - -#### Request - -```json -PUT _plugins/_anomaly_detection/detectors/ -{ - "name": "test-detector", - "description": "Test detector", - "time_field": "timestamp", - "indices": [ - "order*" - ], - "feature_attributes": [ - { - "feature_name": "total_order", - "feature_enabled": true, - "aggregation_query": { - "total_order": { - "sum": { - "field": "value" - } - } - } - } - ], - "filter_query": { - "bool": { - "filter": [ - { - "exists": { - "field": "value", - "boost": 1 - } - } - ], - "adjust_pure_negative": true, - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 10, - "unit": "MINUTES" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "MINUTES" - } - } -} -``` - - -#### Sample response - -```json -{ - "_id" : "m4ccEnIBTXsGi3mvMt9p", - "_version" : 2, - "_seq_no" : 4, - "_primary_term" : 1, - "anomaly_detector" : { - "name" : "test-detector", - "description" : "Test detector", - "time_field" : "timestamp", - "indices" : [ - "order*" - ], - "filter_query" : { - "bool" : { - "filter" : [ - { - "exists" : { - "field" : "value", - "boost" : 1.0 - } - } - ], - "adjust_pure_negative" : true, - "boost" : 1.0 - } - }, - "detection_interval" : { - "period" : { - "interval" : 10, - "unit" : "Minutes" - } - }, - "window_delay" : { - "period" : { - "interval" : 1, - "unit" : "Minutes" - } - }, - "schema_version" : 0, - "feature_attributes" : [ - { - "feature_id" : "xxokEnIBcpeWMD987A1X", - "feature_name" : "total_order", - "feature_enabled" : true, - "aggregation_query" : { - "total_order" : { - "sum" : { - "field" : "value" - } - } - } - } - ] - } -} -``` - ---- - -## Get detector -Introduced 1.0 -{: .label .label-purple } - -Returns all information about a detector based on the `detector_id`. - -#### Request - -```json -GET _plugins/_anomaly_detection/detectors/ -``` - -#### Sample response - -```json -{ - "_id" : "m4ccEnIBTXsGi3mvMt9p", - "_version" : 1, - "_primary_term" : 1, - "_seq_no" : 3, - "anomaly_detector" : { - "name" : "test-detector", - "description" : "Test detector", - "time_field" : "timestamp", - "indices" : [ - "order*" - ], - "filter_query" : { - "bool" : { - "filter" : [ - { - "exists" : { - "field" : "value", - "boost" : 1.0 - } - } - ], - "adjust_pure_negative" : true, - "boost" : 1.0 - } - }, - "detection_interval" : { - "period" : { - "interval" : 1, - "unit" : "Minutes" - } - }, - "window_delay" : { - "period" : { - "interval" : 1, - "unit" : "Minutes" - } - }, - "schema_version" : 0, - "feature_attributes" : [ - { - "feature_id" : "mYccEnIBTXsGi3mvMd8_", - "feature_name" : "total_order", - "feature_enabled" : true, - "aggregation_query" : { - "total_order" : { - "sum" : { - "field" : "value" - } - } - } - } - ], - "last_update_time" : 1589441737319 - } -} -``` - - -Use `job=true` to get anomaly detection job information. - -#### Request - -```json -GET _plugins/_anomaly_detection/detectors/?job=true -``` - -#### Sample response - -```json -{ - "_id": "LJxGsXcBoDQA8W1Q--A1", - "_version": 1, - "_primary_term": 1, - "_seq_no": 0, - "anomaly_detector": { - "name": "test2", - "description": "test", - "time_field": "timestamp", - "indices": [ - "server_log" - ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "K5xGsXcBoDQA8W1Q-uCF", - "feature_name": "F1", - "feature_enabled": "true", - "aggregation_query": { - "f_1": { - "sum": { - "field": "value" - } - } - } - } - ], - "last_update_time": 1613586955060, - "detector_type": "MULTI_ENTITY" - }, - "anomaly_detector_job": { - "name": "LJxGsXcBoDQA8W1Q--A1", - "schedule": { - "interval": { - "start_time": 1613587220387, - "period": 1, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "enabled": "false", - "enabled_time": 1613587220387, - "last_update_time": 1613587289169, - "lock_duration_seconds": 60, - "disabled_time": 1613587289169 - }, - "anomaly_detection_task": { - "task_id": "WZ5LsXcBoDQA8W1QmUa3", - "last_update_time": 1613587349022, - "error": "Task cancelled by user", - "state": "STOPPED", - "detector_id": "LJxGsXcBoDQA8W1Q--A1", - "task_progress": 0.26321793, - "init_progress": 1, - "current_piece": 1611030900000, - "execution_start_time": 1613587257783, - "execution_end_time": 1613587349022, - "is_latest": "true", - "task_type": "HISTORICAL", - "coordinating_node": "NSw5j-3YQeGkH8KESVKlzw", - "worker_node": "NSw5j-3YQeGkH8KESVKlzw", - "detector": { - "name": "test2", - "description": "test", - "time_field": "timestamp", - "indices": [ - "server_log" - ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "K5xGsXcBoDQA8W1Q-uCF", - "feature_name": "F1", - "feature_enabled": "true", - "aggregation_query": { - "f_1": { - "sum": { - "field": "value" - } - } - } - } - ], - "last_update_time": 1613586955060, - "detector_type": "MULTI_ENTITY" - } - } -} -``` - -Use `task=true` to get historical analysis task information. - -#### Request - -```json -GET _plugins/_anomaly_detection/detectors/?task=true -``` - -#### Sample response - -```json -{ - "_id": "BwzKQXcB89DLS7G9rg7Y", - "_version": 1, - "_primary_term": 2, - "_seq_no": 10, - "anomaly_detector": { - "name": "test-ylwu1", - "description": "test", - "time_field": "timestamp", - "indices": [ - "ser*" - ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 10, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "BgzKQXcB89DLS7G9rg7G", - "feature_name": "F1", - "feature_enabled": true, - "aggregation_query": { - "f_1": { - "sum": { - "field": "value" - } - } - } - } - ], - "ui_metadata": { - "features": { - "F1": { - "aggregationBy": "sum", - "aggregationOf": "value", - "featureType": "simple_aggs" - } - } - }, - "last_update_time": 1611716538071, - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "all_access", - "own_index" - ], - "custom_attribute_names": [], - "user_requested_tenant": "__user__" - }, - "detector_type": "HISTORICAL_SINGLE_ENTITY", - "detection_date_range": { - "start_time": 1580094137997, - "end_time": 1611716537997 - } - }, - "anomaly_detection_task": { - "task_id": "sgxaRXcB89DLS7G9RfIO", - "last_update_time": 1611776648699, - "started_by": "admin", - "state": "FINISHED", - "detector_id": "BwzKQXcB89DLS7G9rg7Y", - "task_progress": 1, - "init_progress": 1, - "current_piece": 1611716400000, - "execution_start_time": 1611776279822, - "execution_end_time": 1611776648679, - "is_latest": true, - "task_type": "HISTORICAL", - "coordinating_node": "gs213KqjS4q7H4Bmn_ZuLA", - "worker_node": "PgfR3JhbT7yJMx7bwQ6E3w", - "detector": { - "name": "test-ylwu1", - "description": "test", - "time_field": "timestamp", - "indices": [ - "ser*" - ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 10, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "BgzKQXcB89DLS7G9rg7G", - "feature_name": "F1", - "feature_enabled": true, - "aggregation_query": { - "f_1": { - "sum": { - "field": "value" - } - } - } - } - ], - "ui_metadata": { - "features": { - "F1": { - "aggregationBy": "sum", - "aggregationOf": "value", - "featureType": "simple_aggs" - } - } - }, - "last_update_time": 1611716538071, - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "all_access", - "own_index" - ], - "custom_attribute_names": [], - "user_requested_tenant": "__user__" - }, - "detector_type": "HISTORICAL_SINGLE_ENTITY", - "detection_date_range": { - "start_time": 1580094137997, - "end_time": 1611716537997 - } - }, - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "all_access", - "own_index" - ], - "custom_attribute_names": [], - "user_requested_tenant": "__user__" - } - } -} -``` --- @@ -2031,7 +2022,7 @@ For historical analysis, you see the values for the following fields: - `ad_canceled_batch_task_count` - `ad_batch_task_failure_count` -For real-time analysis, these values are 0. +If haven't run any historical analysis, these values show up as 0. #### Sample response @@ -3237,7 +3228,7 @@ GET _plugins/_anomaly_detection/detectors//_profile?_all=true } ``` -For historical analysis, specify `_all` or `ad_task` to see information about its latest task: +To get profile information for only historical analysis, specify `ad_task`: #### Request diff --git a/_monitoring-plugins/ad/settings.md b/_monitoring-plugins/ad/settings.md index eaebdee2..5aa73dc4 100644 --- a/_monitoring-plugins/ad/settings.md +++ b/_monitoring-plugins/ad/settings.md @@ -35,13 +35,13 @@ Setting | Default | Description `plugins.anomaly_detection.max_entities_for_preview` | 5 | The maximum unique category field values displayed with the preview operation for high cardinality detectors. By default, if the category field(s) have more than the configured unique values in a detector interval, the anomaly detection plugin orders them by the natural ordering of categorical values (for example, entity `ab` comes before `bc`) and then selects the top values. `plugins.anomaly_detection.max_primary_shards` | 10 | The maximum number of primary shards an anomaly detection index can have. `plugins.anomaly_detection.filter_by_backend_roles` | False | When you enable the security plugin and set this to `true`, the anomaly detection plugin filters results based on the user's backend role(s). -`plugins.anomaly_detection.max_batch_task_per_node` | 10 | Starting a historical detector triggers a batch task. This setting is the number of batch tasks that you can run per data node. You can tune this setting from 1 to 1,000. If the data nodes can't support all batch tasks and you're not sure if the data nodes are capable of running more historical detectors, add more data nodes instead of changing this setting to a higher value. -`plugins.anomaly_detection.max_old_ad_task_docs_per_detector` | 1 | You can run the same historical detector many times. For each run, the anomaly detection plugin creates a new task. This setting is the number of previous tasks the plugin keeps. Set this value to at least 1 to track its last run. You can keep a maximum of 1,000 old tasks to avoid overwhelming the cluster. +`plugins.anomaly_detection.max_batch_task_per_node` | 10 | Starting a historical analysis triggers a batch task. This setting is the number of batch tasks that you can run per data node. You can tune this setting from 1 to 1,000. If the data nodes can’t support all batch tasks and you’re not sure if the data nodes are capable of running more historical analysis, add more data nodes instead of changing this setting to a higher value. Increasing this value might bring more load on each data node. +`plugins.anomaly_detection.max_old_ad_task_docs_per_detector` | 1 | You can run historical analysis for the same detector many times. For each run, the anomaly detection plugin creates a new task. This setting is the number of previous tasks the plugin keeps. Set this value to at least 1 to track its last run. You can keep a maximum of 1,000 old tasks to avoid overwhelming the cluster. `plugins.anomaly_detection.batch_task_piece_size` | 1,000 | The date range for a historical task is split into smaller pieces and the anomaly detection plugin runs the task piece by piece. Each piece contains 1,000 detection intervals by default. For example, if detector interval is 1 minute and one piece is 1,000 minutes, the feature data is queried every 1,000 minutes. You can change this setting from 1 to 10,000. -`plugins.anomaly_detection.batch_task_piece_interval_seconds` | 5 | Add a time interval between historical detector tasks. This interval prevents the task from consuming too much of the available resources and starving other operations like search and bulk index. You can change this setting from 1 to 600 seconds. -`plugins.anomaly_detection.max_top_entities_for_historical_analysis` | 1,000 | The maximum number of top entities that you run for a high cardinality detector historical analysis. -`plugins.anomaly_detection.max_running_entities_per_detector_for_historical_analysis` | 10 | The number of entity tasks that you can run in parallel for a single high cardinality detector. The task slots available on your cluster also impact how many entities run in parallel. If a cluster has 3 data nodes, each data node has 10 task slots by default. Say you already have two high cardinality detectors and each of them run 10 entities. If you start a single-flow detector that takes 1 task slot, the number of task slots available is 10 * 3 - 10 * 2 - 1 = 9. if you now start a new high cardinality detector, the detector can only run 9 entities in parallel and not 10. -`plugins.anomaly_detection.max_cached_deleted_tasks` | 1,000 | You can rerun historical analysis for a single detector as many times as you like. The anomaly detection plugin only keeps a limited number of old tasks, by default 1 old task. If you run historical analysis three times for a detector, the oldest task is deleted. Because historical analysis generates a number of anomaly results in a short span of time, it's necessary to clean up anomaly results for a deleted task. With this field, you can configure how many deleted tasks you can cache at most. The plugin cleans up a task's results when it's deleted. If the plugin fails to do this cleanup, it adds the task's results into a cache and an hourly cron job performs the cleanup. After an hour, if still you find an old task result in the cache, use the [delete detector results API]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/api/#delete-detector-results) to delete the task result manually. +`plugins.anomaly_detection.batch_task_piece_interval_seconds` | 5 | Add a time interval between two pieces of the same historical analysis task. This interval prevents the task from consuming too much of the available resources and starving other operations like search and bulk index. You can change this setting from 1 to 600 seconds. +`plugins.anomaly_detection.max_top_entities_for_historical_analysis` | 1,000 | The maximum number of top entities that you run for a high cardinality detector historical analysis. The range is from 1 to 10,000. +`plugins.anomaly_detection.max_running_entities_per_detector_for_historical_analysis` | 10 | The number of entity tasks that you can run in parallel for a single high cardinality detector. The task slots available on your cluster also impact how many entities run in parallel. If a cluster has 3 data nodes, each data node has 10 task slots by default. Say you already have two high cardinality detectors and each of them run 10 entities. If you start a single-flow detector that takes 1 task slot, the number of task slots available is 10 * 3 - 10 * 2 - 1 = 9. if you now start a new high cardinality detector, the detector can only run 9 entities in parallel and not 10. You can tune this value from 1 to 1,000 based on your cluster's capability. If you set a higher value, the anomaly detection plugin runs historical analysis faster but also consumes more resources. +`plugins.anomaly_detection.max_cached_deleted_tasks` | 1,000 | You can rerun historical analysis for a single detector as many times as you like. The anomaly detection plugin only keeps a limited number of old tasks, by default 1 old task. If you run historical analysis three times for a detector, the oldest task is deleted. Because historical analysis generates a number of anomaly results in a short span of time, it's necessary to clean up anomaly results for a deleted task. With this field, you can configure how many deleted tasks you can cache at most. The plugin cleans up a task's results when it's deleted. If the plugin fails to do this cleanup, it adds the task's results into a cache and an hourly cron job performs the cleanup. You can use this setting to limit how many old tasks are put into cache to avoid a DDoS attack. After an hour, if still you find an old task result in the cache, use the [delete detector results API]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/api/#delete-detector-results) to delete the task result manually. You can tune this setting from 1 to 10,000. `plugins.anomaly_detection.delete_anomaly_result_when_delete_detector` | False | Whether the anomaly detection plugin deletes the anomaly result when you delete a detector. If you want to save some disk space, especially if you've high cardinality detectors generating a lot of results, set this field to true. Alternatively, you can use the [delete detector results API]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/api/#delete-detector-results) to manually delete the results. `plugins.anomaly_detection.dedicated_cache_size` | 10 | If the real-time analysis of a high cardinality detector starts successfully, the anomaly detection plugin guarantees keeping 10 (dynamically adjustable via this setting) entities' models in memory per node. If the number of entities exceeds this limit, the plugin puts the extra entities' models in a memory space shared by all detectors. The actual number of entities varies based on the memory that you've available and the frequencies of the entities. If you'd like the plugin to guarantee keeping more entities' models in memory and if you're cluster has sufficient memory, you can increase this setting value. `plugins.anomaly_detection.max_concurrent_preview` | 2 | The maximum number of concurrent previews. You can use this setting to limit resource usage. From f3897b621cde2bc812f55d4eab3bdf739fe92339 Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Mon, 4 Oct 2021 14:18:04 -0700 Subject: [PATCH 066/229] minor change --- _monitoring-plugins/ad/api.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/_monitoring-plugins/ad/api.md b/_monitoring-plugins/ad/api.md index b6aaefe5..57dadd6f 100644 --- a/_monitoring-plugins/ad/api.md +++ b/_monitoring-plugins/ad/api.md @@ -3186,7 +3186,7 @@ For a single-entity detector: The `total_entities` parameter shows you the total number of entities including the number of category fields for a detector. -Getting the total count of entities is an expensive operation for a detector with more than one category field. By default, a real-time detector counts the number of entities up to a value of 10,000 and historical analysis counts the number of entities up to a value of 1,000. +Getting the total count of entities is an expensive operation for real-time analysis of a detector with more than one category field. By default, for a real-time detection profile, a detector counts the number of entities up to a value of 10,000 and historical analysis counts the number of entities up to a value of 1,000. The `profile` operation also provides information about each entity, such as the entity’s `last_sample_timestamp` and `last_active_timestamp`. `last_sample_timestamp` shows the last document in the input data source index containing the entity, while `last_active_timestamp` shows the timestamp when the entity’s model was last seen in the model cache. @@ -3228,7 +3228,8 @@ GET _plugins/_anomaly_detection/detectors//_profile?_all=true } ``` -To get profile information for only historical analysis, specify `ad_task`: +To get profile information for only historical analysis, specify `ad_task`. +Specifying `_all` is an expensive operation for multi-category high cardinality detectors. #### Request From 074f1601978aced7cca9669045aa81adbe02cc2c Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Mon, 4 Oct 2021 14:21:36 -0700 Subject: [PATCH 067/229] minor fix --- _monitoring-plugins/ad/api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_monitoring-plugins/ad/api.md b/_monitoring-plugins/ad/api.md index 57dadd6f..f99825fd 100644 --- a/_monitoring-plugins/ad/api.md +++ b/_monitoring-plugins/ad/api.md @@ -2790,7 +2790,7 @@ GET _plugins/_anomaly_detection/detectors//_profile/total_size_in_by You can see the `ad_task` field only for historical analysis. -The `model_count` parameter shows the total number of models that a detector runs in memory. This is useful if you have several models running on your cluster and want to know the count. +The `model_count` parameter shows the total number of models that a detector runs on each node’s memory. This is useful if you have several models running on your cluster and want to know the count. If you configured the category field, you can see the number of unique values in the field and all active entities with models running in memory. @@ -3186,7 +3186,7 @@ For a single-entity detector: The `total_entities` parameter shows you the total number of entities including the number of category fields for a detector. -Getting the total count of entities is an expensive operation for real-time analysis of a detector with more than one category field. By default, for a real-time detection profile, a detector counts the number of entities up to a value of 10,000 and historical analysis counts the number of entities up to a value of 1,000. +Getting the total count of entities is an expensive operation for real-time analysis of a detector with more than one category field. By default, for a real-time detection profile, a detector counts the number of entities up to a value of 10,000. For historical analysis, the anomaly detection plugin only detects the top 1,000 entities by default and caches the top entities in memory to reduce the cost of getting the total count of entities for historical analysis. The `profile` operation also provides information about each entity, such as the entity’s `last_sample_timestamp` and `last_active_timestamp`. `last_sample_timestamp` shows the last document in the input data source index containing the entity, while `last_active_timestamp` shows the timestamp when the entity’s model was last seen in the model cache. From f57469e0f79284bc6de4dd6e0e62ff58d3b0b06f Mon Sep 17 00:00:00 2001 From: Liz Snyder Date: Mon, 4 Oct 2021 17:08:58 -0700 Subject: [PATCH 068/229] Add back Docker sample --- _replication-plugin/auto-follow.md | 2 +- _replication-plugin/get-started.md | 16 +++------------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/_replication-plugin/auto-follow.md b/_replication-plugin/auto-follow.md index 4af6c61f..fd2e69b4 100644 --- a/_replication-plugin/auto-follow.md +++ b/_replication-plugin/auto-follow.md @@ -48,7 +48,7 @@ If the security plugin is disabled, you can leave out the `use_roles` parameter. To test the rule, create a matching index on the leader cluster: ```bash -curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/movies-0001' +curl -XPUT -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9201/movies-0001?pretty' ``` And confirm its replica shows up on the follower cluster: diff --git a/_replication-plugin/get-started.md b/_replication-plugin/get-started.md index e6b3c36a..2ab9d755 100644 --- a/_replication-plugin/get-started.md +++ b/_replication-plugin/get-started.md @@ -32,9 +32,6 @@ plugins.security.nodes_dn_dynamic_config_enabled: true ## Example setup -The following example demonstrates how to replicate data between two single-node clusters: `leader-cluster` on port 9201, and `follower-cluster` on port 9200. -{% comment %} - Save this sample file as `docker-compose.yml` and run `docker-compose up` to start two single-node clusters on the same network: ```yml @@ -89,26 +86,20 @@ networks: After the clusters start, verify the names of each: -{% endcomment %} - ```bash curl -XGET -u 'admin:admin' -k 'https://localhost:9201' { - "name" : "replication-node1", "cluster_name" : "leader-cluster", ... } curl -XGET -u 'admin:admin' -k 'https://localhost:9200' { - "name" : "replication-node2", "cluster_name" : "follower-cluster", ... } ``` -{% comment %} - For this example, use port 9201 (`replication-node1`) as the leader and port 9200 (`replication-node2`) as the follower cluster. To get the IP address for the leader cluster, first identify its container ID: @@ -116,8 +107,8 @@ To get the IP address for the leader cluster, first identify its container ID: ```bash docker ps CONTAINER ID IMAGE PORTS NAMES -3b8cdc698be5 opensearchproject/opensearch:{{site.opensearch_version}} 0.0.0.0:9200->9200/tcp, 0.0.0.0:9600->9600/tcp, 9300/tcp replication-node1 -731f5e8b0f4b opensearchproject/opensearch:{{site.opensearch_version}} 9300/tcp, 0.0.0.0:9201->9200/tcp, 0.0.0.0:9700->9600/tcp replication-node2 +3b8cdc698be5 opensearchproject/opensearch:{{site.opensearch_version}} 0.0.0.0:9200->9200/tcp, 0.0.0.0:9600->9600/tcp, 9300/tcp replication-node2 +731f5e8b0f4b opensearchproject/opensearch:{{site.opensearch_version}} 9300/tcp, 0.0.0.0:9201->9200/tcp, 0.0.0.0:9700->9600/tcp replication-node1 ``` Then get that container's IP address: @@ -126,7 +117,6 @@ Then get that container's IP address: docker inspect --format='{% raw %}{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}{% endraw %}' 731f5e8b0f4b 172.22.0.3 ``` -{% endcomment %} ## Set up a cross-cluster connection @@ -261,7 +251,7 @@ When replication resumes, the follower index picks up any changes that were made Terminate replication of a specified index from the follower cluster: ```bash -curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_stop' -d '{}' +curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/follower-01/_stop?pretty' -d '{}' ``` When you stop replication, the follower index un-follows the leader and becomes a standard index that you can write to. You can't restart replication after stopping it. From a23cdb91aeced6df567216c70143b8c880e48c6e Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Tue, 5 Oct 2021 02:36:58 -0700 Subject: [PATCH 069/229] updated requests and responses --- _monitoring-plugins/ad/api.md | 4237 +++++++++++++--------------- _monitoring-plugins/ad/index.md | 2 +- _monitoring-plugins/ad/settings.md | 2 +- 3 files changed, 2032 insertions(+), 2209 deletions(-) diff --git a/_monitoring-plugins/ad/api.md b/_monitoring-plugins/ad/api.md index f99825fd..46c12c28 100644 --- a/_monitoring-plugins/ad/api.md +++ b/_monitoring-plugins/ad/api.md @@ -24,8 +24,7 @@ Introduced 1.0 Creates an anomaly detector. -This command creates a detector named `test-detector` that finds anomalies based on the sum of the `value` field: - +This command creates a single-flow detector named `test-detector` that finds anomalies based on the sum of the `value` field: #### Request @@ -36,14 +35,14 @@ POST _plugins/_anomaly_detection/detectors "description": "Test detector", "time_field": "timestamp", "indices": [ - "order*" + "server_log*" ], "feature_attributes": [ { - "feature_name": "total_order", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "total_order": { + "test": { "sum": { "field": "value" } @@ -55,9 +54,10 @@ POST _plugins/_anomaly_detection/detectors "bool": { "filter": [ { - "exists": { - "field": "value", - "boost": 1 + "range": { + "value": { + "gt": 1 + } } } ], @@ -84,24 +84,28 @@ POST _plugins/_anomaly_detection/detectors ```json { - "_id": "m4ccEnIBTXsGi3mvMt9p", + "_id": "VEHKTXwBwf_U8gjUXY2s", "_version": 1, - "_seq_no": 3, - "_primary_term": 1, + "_seq_no": 5, "anomaly_detector": { "name": "test-detector", "description": "Test detector", "time_field": "timestamp", "indices": [ - "order*" + "server_log*" ], "filter_query": { "bool": { "filter": [ { - "exists": { - "field": "value", - "boost": 1 + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } } } ], @@ -121,66 +125,97 @@ POST _plugins/_anomaly_detection/detectors "unit": "Minutes" } }, + "shingle_size": 8, "schema_version": 0, "feature_attributes": [ { - "feature_id": "mYccEnIBTXsGi3mvMd8_", - "feature_name": "total_order", + "feature_id": "U0HKTXwBwf_U8gjUXY2m", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "total_order": { + "test": { "sum": { "field": "value" } } } } - ] - } + ], + "last_update_time": 1633392680364, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "SINGLE_ENTITY" + }, + "_primary_term": 1 } ``` -To set a category field for high cardinality: +To create high cardinality detector by specifying a category field: #### Request ```json POST _plugins/_anomaly_detection/detectors { - "name": "Host OK Rate Detector", - "description": "ok rate", - "time_field": "@timestamp", + "name": "test-hc-detector", + "description": "Test detector", + "time_field": "timestamp", "indices": [ - "host-cloudwatch" - ], - "category_field": [ - "host" + "server_log*" ], "feature_attributes": [ { - "feature_name": "latency_max", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "latency_max": { - "max": { - "field": "latency" + "test": { + "sum": { + "field": "value" } } } } ], - "window_delay": { - "period": { - "interval": 10, - "unit": "MINUTES" + "filter_query": { + "bool": { + "filter": [ + { + "range": { + "value": { + "gt": 1 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1 } }, "detection_interval": { "period": { "interval": 1, - "unit": "MINUTES" + "unit": "Minutes" } - } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "category_field": [ + "ip" + ] } ``` @@ -188,18 +223,32 @@ POST _plugins/_anomaly_detection/detectors ```json { - "_id": "4CIGoHUBTpMGN-4KzBQg", + "_id": "b0HRTXwBwf_U8gjUw43R", "_version": 1, - "_seq_no": 0, + "_seq_no": 6, "anomaly_detector": { - "name": "Host OK Rate Detector", - "description": "ok rate", - "time_field": "@timestamp", + "name": "test-hc-detector", + "description": "Test detector", + "time_field": "timestamp", "indices": [ - "server-metrics" + "server_log*" ], "filter_query": { - "match_all": { + "bool": { + "filter": [ + { + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } + } + } + ], + "adjust_pure_negative": true, "boost": 1 } }, @@ -211,35 +260,62 @@ POST _plugins/_anomaly_detection/detectors }, "window_delay": { "period": { - "interval": 10, - "unit": "MINUTES" + "interval": 1, + "unit": "Minutes" } }, - "shingle_size": 1, - "schema_version": 2, + "shingle_size": 8, + "schema_version": 0, "feature_attributes": [ { - "feature_id": "0Kld3HUBhpHMyt2e_UHn", - "feature_name": "latency_max", + "feature_id": "bkHRTXwBwf_U8gjUw43K", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "latency_max": { - "max": { - "field": "latency" + "test": { + "sum": { + "field": "value" } } } } ], - "last_update_time": 1604707601438, + "last_update_time": 1633393165265, "category_field": [ - "host" - ] + "ip" + ], + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "MULTI_ENTITY" }, "_primary_term": 1 } ``` +You can specify a maximum of two category fields: + +```json +"category_field": [ + "ip" +] +``` + +```json +"category_field": [ + "ip", "error_type" +] +``` + You can specify the following options. Options | Description | Type | Required @@ -272,65 +348,83 @@ GET _plugins/_anomaly_detection/detectors/ ```json { - "_id" : "m4ccEnIBTXsGi3mvMt9p", - "_version" : 1, - "_primary_term" : 1, - "_seq_no" : 3, - "anomaly_detector" : { - "name" : "test-detector", - "description" : "Test detector", - "time_field" : "timestamp", - "indices" : [ - "order*" + "_id": "VEHKTXwBwf_U8gjUXY2s", + "_version": 1, + "_primary_term": 1, + "_seq_no": 5, + "anomaly_detector": { + "name": "test-detector", + "description": "Test detector", + "time_field": "timestamp", + "indices": [ + "server_log*" ], - "filter_query" : { - "bool" : { - "filter" : [ + "filter_query": { + "bool": { + "filter": [ { - "exists" : { - "field" : "value", - "boost" : 1.0 + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } } } ], - "adjust_pure_negative" : true, - "boost" : 1.0 + "adjust_pure_negative": true, + "boost": 1 } }, - "detection_interval" : { - "period" : { - "interval" : 1, - "unit" : "Minutes" + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" } }, - "window_delay" : { - "period" : { - "interval" : 1, - "unit" : "Minutes" + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" } }, - "schema_version" : 0, - "feature_attributes" : [ + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ { - "feature_id" : "mYccEnIBTXsGi3mvMd8_", - "feature_name" : "total_order", - "feature_enabled" : true, - "aggregation_query" : { - "total_order" : { - "sum" : { - "field" : "value" + "feature_id": "U0HKTXwBwf_U8gjUXY2m", + "feature_name": "test", + "feature_enabled": true, + "aggregation_query": { + "test": { + "sum": { + "field": "value" } } } } ], - "last_update_time" : 1589441737319 + "last_update_time": 1633392680364, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "SINGLE_ENTITY" } } ``` - -Use `job=true` to get anomaly detection job information. +Use `task=true` to get real-time analysis task information. #### Request @@ -342,19 +436,33 @@ GET _plugins/_anomaly_detection/detectors/?job=true ```json { - "_id": "LJxGsXcBoDQA8W1Q--A1", + "_id": "VEHKTXwBwf_U8gjUXY2s", "_version": 1, "_primary_term": 1, - "_seq_no": 0, + "_seq_no": 5, "anomaly_detector": { - "name": "test2", - "description": "test", + "name": "test-detector", + "description": "Test detector", "time_field": "timestamp", "indices": [ - "server_log" + "server_log*" ], "filter_query": { - "match_all": { + "bool": { + "filter": [ + { + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } + } + } + ], + "adjust_pure_negative": true, "boost": 1 } }, @@ -374,11 +482,11 @@ GET _plugins/_anomaly_detection/detectors/?job=true "schema_version": 0, "feature_attributes": [ { - "feature_id": "K5xGsXcBoDQA8W1Q-uCF", - "feature_name": "F1", - "feature_enabled": "true", + "feature_id": "U0HKTXwBwf_U8gjUXY2m", + "feature_name": "test", + "feature_enabled": true, "aggregation_query": { - "f_1": { + "test": { "sum": { "field": "value" } @@ -386,14 +494,26 @@ GET _plugins/_anomaly_detection/detectors/?job=true } } ], - "last_update_time": 1613586955060, - "detector_type": "MULTI_ENTITY" + "last_update_time": 1633392680364, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "SINGLE_ENTITY" }, "anomaly_detector_job": { - "name": "LJxGsXcBoDQA8W1Q--A1", + "name": "VEHKTXwBwf_U8gjUXY2s", "schedule": { "interval": { - "start_time": 1613587220387, + "start_time": 1633393656357, "period": 1, "unit": "Minutes" } @@ -404,75 +524,28 @@ GET _plugins/_anomaly_detection/detectors/?job=true "unit": "Minutes" } }, - "enabled": "false", - "enabled_time": 1613587220387, - "last_update_time": 1613587289169, + "enabled": true, + "enabled_time": 1633393656357, + "last_update_time": 1633393656357, "lock_duration_seconds": 60, - "disabled_time": 1613587289169 - }, - "anomaly_detection_task": { - "task_id": "WZ5LsXcBoDQA8W1QmUa3", - "last_update_time": 1613587349022, - "error": "Task cancelled by user", - "state": "STOPPED", - "detector_id": "LJxGsXcBoDQA8W1Q--A1", - "task_progress": 0.26321793, - "init_progress": 1, - "current_piece": 1611030900000, - "execution_start_time": 1613587257783, - "execution_end_time": 1613587349022, - "is_latest": "true", - "task_type": "HISTORICAL", - "coordinating_node": "NSw5j-3YQeGkH8KESVKlzw", - "worker_node": "NSw5j-3YQeGkH8KESVKlzw", - "detector": { - "name": "test2", - "description": "test", - "time_field": "timestamp", - "indices": [ - "server_log" + "user": { + "name": "admin", + "backend_roles": [ + "admin" ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "K5xGsXcBoDQA8W1Q-uCF", - "feature_name": "F1", - "feature_enabled": "true", - "aggregation_query": { - "f_1": { - "sum": { - "field": "value" - } - } - } - } + "roles": [ + "own_index", + "all_access" ], - "last_update_time": 1613586955060, - "detector_type": "MULTI_ENTITY" + "custom_attribute_names": [], + "user_requested_tenant": "__user__" } } } ``` Use `task=true` to get historical analysis task information. +You can set both `job=true` and `task=true` to get information for both real-time and historical analysis tasks. #### Request @@ -484,25 +557,39 @@ GET _plugins/_anomaly_detection/detectors/?task=true ```json { - "_id": "BwzKQXcB89DLS7G9rg7Y", + "_id": "VEHKTXwBwf_U8gjUXY2s", "_version": 1, - "_primary_term": 2, - "_seq_no": 10, + "_primary_term": 1, + "_seq_no": 5, "anomaly_detector": { - "name": "test-ylwu1", - "description": "test", + "name": "test-detector", + "description": "Test detector", "time_field": "timestamp", "indices": [ - "ser*" + "server_log*" ], "filter_query": { - "match_all": { + "bool": { + "filter": [ + { + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } + } + } + ], + "adjust_pure_negative": true, "boost": 1 } }, "detection_interval": { "period": { - "interval": 10, + "interval": 1, "unit": "Minutes" } }, @@ -516,11 +603,11 @@ GET _plugins/_anomaly_detection/detectors/?task=true "schema_version": 0, "feature_attributes": [ { - "feature_id": "BgzKQXcB89DLS7G9rg7G", - "feature_name": "F1", + "feature_id": "U0HKTXwBwf_U8gjUXY2m", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "f_1": { + "test": { "sum": { "field": "value" } @@ -528,64 +615,63 @@ GET _plugins/_anomaly_detection/detectors/?task=true } } ], - "ui_metadata": { - "features": { - "F1": { - "aggregationBy": "sum", - "aggregationOf": "value", - "featureType": "simple_aggs" - } - } - }, - "last_update_time": 1611716538071, + "last_update_time": 1633392680364, "user": { "name": "admin", "backend_roles": [ "admin" ], "roles": [ - "all_access", - "own_index" + "own_index", + "all_access" ], "custom_attribute_names": [], "user_requested_tenant": "__user__" }, - "detector_type": "HISTORICAL_SINGLE_ENTITY", - "detection_date_range": { - "start_time": 1580094137997, - "end_time": 1611716537997 - } + "detector_type": "SINGLE_ENTITY" }, - "anomaly_detection_task": { - "task_id": "sgxaRXcB89DLS7G9RfIO", - "last_update_time": 1611776648699, + "realtime_detection_task": { + "task_id": "nkTZTXwBjd8s6RK4QlMq", + "last_update_time": 1633393776375, "started_by": "admin", - "state": "FINISHED", - "detector_id": "BwzKQXcB89DLS7G9rg7Y", - "task_progress": 1, + "error": "", + "state": "RUNNING", + "detector_id": "VEHKTXwBwf_U8gjUXY2s", + "task_progress": 0, "init_progress": 1, - "current_piece": 1611716400000, - "execution_start_time": 1611776279822, - "execution_end_time": 1611776648679, + "execution_start_time": 1633393656362, "is_latest": true, - "task_type": "HISTORICAL", - "coordinating_node": "gs213KqjS4q7H4Bmn_ZuLA", - "worker_node": "PgfR3JhbT7yJMx7bwQ6E3w", + "task_type": "REALTIME_SINGLE_ENTITY", + "coordinating_node": "SWD7ihu9TaaW1zKwFZNVNg", "detector": { - "name": "test-ylwu1", - "description": "test", + "name": "test-detector", + "description": "Test detector", "time_field": "timestamp", "indices": [ - "ser*" + "server_log*" ], "filter_query": { - "match_all": { + "bool": { + "filter": [ + { + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } + } + } + ], + "adjust_pure_negative": true, "boost": 1 } }, "detection_interval": { "period": { - "interval": 10, + "interval": 1, "unit": "Minutes" } }, @@ -599,11 +685,11 @@ GET _plugins/_anomaly_detection/detectors/?task=true "schema_version": 0, "feature_attributes": [ { - "feature_id": "BgzKQXcB89DLS7G9rg7G", - "feature_name": "F1", + "feature_id": "U0HKTXwBwf_U8gjUXY2m", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "f_1": { + "test": { "sum": { "field": "value" } @@ -611,33 +697,121 @@ GET _plugins/_anomaly_detection/detectors/?task=true } } ], - "ui_metadata": { - "features": { - "F1": { - "aggregationBy": "sum", - "aggregationOf": "value", - "featureType": "simple_aggs" - } - } - }, - "last_update_time": 1611716538071, + "last_update_time": 1633392680364, "user": { "name": "admin", "backend_roles": [ "admin" ], "roles": [ - "all_access", - "own_index" + "own_index", + "all_access" ], "custom_attribute_names": [], "user_requested_tenant": "__user__" }, - "detector_type": "HISTORICAL_SINGLE_ENTITY", - "detection_date_range": { - "start_time": 1580094137997, - "end_time": 1611716537997 - } + "detector_type": "SINGLE_ENTITY" + }, + "estimated_minutes_left": 0, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + } + }, + "historical_analysis_task": { + "task_id": "99DaTXwB6HknB84StRN1", + "last_update_time": 1633393797040, + "started_by": "admin", + "state": "RUNNING", + "detector_id": "VEHKTXwBwf_U8gjUXY2s", + "task_progress": 0.89285713, + "init_progress": 1, + "current_piece": 1633328940000, + "execution_start_time": 1633393751412, + "is_latest": true, + "task_type": "HISTORICAL_SINGLE_ENTITY", + "coordinating_node": "SWD7ihu9TaaW1zKwFZNVNg", + "worker_node": "2Z4q22BySEyzakYt_A0A2A", + "detector": { + "name": "test-detector", + "description": "Test detector", + "time_field": "timestamp", + "indices": [ + "server_log*" + ], + "filter_query": { + "bool": { + "filter": [ + { + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "U0HKTXwBwf_U8gjUXY2m", + "feature_name": "test", + "feature_enabled": true, + "aggregation_query": { + "test": { + "sum": { + "field": "value" + } + } + } + } + ], + "last_update_time": 1633392680364, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "SINGLE_ENTITY" + }, + "detection_date_range": { + "start_time": 1632788951329, + "end_time": 1633393751329 }, "user": { "name": "admin", @@ -645,8 +819,8 @@ GET _plugins/_anomaly_detection/detectors/?task=true "admin" ], "roles": [ - "all_access", - "own_index" + "own_index", + "all_access" ], "custom_attribute_names": [], "user_requested_tenant": "__user__" @@ -662,7 +836,10 @@ Introduced 1.0 {: .label .label-purple } Updates a detector with any changes, including the description or adding or removing of features. -To update a detector, you need to first stop the detector. +To update a detector, you need to first stop both real-time detection and historical analysis. + +You can't update a category field. +{: .note } #### Request @@ -670,17 +847,17 @@ To update a detector, you need to first stop the detector. PUT _plugins/_anomaly_detection/detectors/ { "name": "test-detector", - "description": "Test detector", + "description": "Test update detector", "time_field": "timestamp", "indices": [ - "order*" + "server_log*" ], "feature_attributes": [ { - "feature_name": "total_order", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "total_order": { + "test": { "sum": { "field": "value" } @@ -692,9 +869,10 @@ PUT _plugins/_anomaly_detection/detectors/ "bool": { "filter": [ { - "exists": { - "field": "value", - "boost": 1 + "range": { + "value": { + "gt": 1 + } } } ], @@ -704,14 +882,14 @@ PUT _plugins/_anomaly_detection/detectors/ }, "detection_interval": { "period": { - "interval": 10, - "unit": "MINUTES" + "interval": 1, + "unit": "Minutes" } }, "window_delay": { "period": { "interval": 1, - "unit": "MINUTES" + "unit": "Minutes" } } } @@ -722,59 +900,114 @@ PUT _plugins/_anomaly_detection/detectors/ ```json { - "_id" : "m4ccEnIBTXsGi3mvMt9p", - "_version" : 2, - "_seq_no" : 4, - "_primary_term" : 1, - "anomaly_detector" : { - "name" : "test-detector", - "description" : "Test detector", - "time_field" : "timestamp", - "indices" : [ - "order*" + "_id": "VEHKTXwBwf_U8gjUXY2s", + "_version": 2, + "_seq_no": 7, + "anomaly_detector": { + "name": "test-detector", + "description": "Test update detector", + "time_field": "timestamp", + "indices": [ + "server_log*" ], - "filter_query" : { - "bool" : { - "filter" : [ + "filter_query": { + "bool": { + "filter": [ { - "exists" : { - "field" : "value", - "boost" : 1.0 + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } } } ], - "adjust_pure_negative" : true, - "boost" : 1.0 + "adjust_pure_negative": true, + "boost": 1 } }, - "detection_interval" : { - "period" : { - "interval" : 10, - "unit" : "Minutes" + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" } }, - "window_delay" : { - "period" : { - "interval" : 1, - "unit" : "Minutes" + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" } }, - "schema_version" : 0, - "feature_attributes" : [ + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ { - "feature_id" : "xxokEnIBcpeWMD987A1X", - "feature_name" : "total_order", - "feature_enabled" : true, - "aggregation_query" : { - "total_order" : { - "sum" : { - "field" : "value" + "feature_id": "3kHiTXwBwf_U8gjUlY15", + "feature_name": "test", + "feature_enabled": true, + "aggregation_query": { + "test": { + "sum": { + "field": "value" } } } } - ] - } + ], + "last_update_time": 1633394267522, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "SINGLE_ENTITY" + }, + "_primary_term": 1 +} +``` + +--- + +## Delete detector +Introduced 1.0 +{: .label .label-purple } + +Deletes a detector based on the `detector_id`. +To delete a detector, you need to first stop both real-time detection and historical analysis. + +#### Request + +```json +DELETE _plugins/_anomaly_detection/detectors/ +``` + +#### Sample response + +```json +{ + "_index": ".opendistro-anomaly-detectors", + "_type": "_doc", + "_id": "70TxTXwBjd8s6RK4j1Pj", + "_version": 2, + "result": "deleted", + "forced_refresh": true, + "_shards": { + "total": 2, + "successful": 2, + "failed": 0 + }, + "_seq_no": 9, + "_primary_term": 1 } ``` @@ -786,20 +1019,114 @@ Introduced 1.0 Passes a date range to the anomaly detector to return any anomalies within that date range. +To preview a single-flow detector: + #### Request ```json POST _plugins/_anomaly_detection/detectors//_preview { - "period_start": 1612982516000, - "period_end": 1614278539000, + "period_start": 1633048868000, + "period_end": 1633394468000, "detector": { "name": "test-detector", - "description": "test server_log", + "description": "Test update detector", "time_field": "timestamp", "indices": [ - "server_log" + "server_log*" ], + "feature_attributes": [ + { + "feature_name": "test", + "feature_enabled": true, + "aggregation_query": { + "test": { + "sum": { + "field": "value" + } + } + } + } + ], + "filter_query": { + "bool": { + "filter": [ + { + "range": { + "value": { + "gt": 1 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + } + } +} +``` + +#### Sample response + +```json +{ + "anomaly_result": [ + { + "detector_id": null, + "data_start_time": 1633049280000, + "data_end_time": 1633049340000, + "schema_version": 0, + "feature_data": [ + { + "feature_id": "8EHmTXwBwf_U8gjU0Y0u", + "feature_name": "test", + "data": 0 + } + ], + "anomaly_grade": 0, + "confidence": 0 + }, + ... + ], + "anomaly_detector": { + "name": "test-detector", + "description": "Test update detector", + "time_field": "timestamp", + "indices": [ + "server_log*" + ], + "filter_query": { + "bool": { + "filter": [ + { + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1 + } + }, "detection_interval": { "period": { "interval": 1, @@ -812,18 +1139,85 @@ POST _plugins/_anomaly_detection/detectors//_preview "unit": "Minutes" } }, + "shingle_size": 8, + "schema_version": 0, "feature_attributes": [ { - "feature_name": "F1", + "feature_id": "8EHmTXwBwf_U8gjU0Y0u", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "f_1": { + "test": { "sum": { "field": "value" } } } } + ], + "detector_type": "SINGLE_ENTITY" + } +} +``` + +If you specify a category field, each result is associated with an entity: + +#### Request + +```json +POST _plugins/_anomaly_detection/detectors//_preview +{ + "period_start": 1633048868000, + "period_end": 1633394468000, + "detector": { + "name": "test-detector", + "description": "Test update detector", + "time_field": "timestamp", + "indices": [ + "server_log*" + ], + "feature_attributes": [ + { + "feature_name": "test", + "feature_enabled": true, + "aggregation_query": { + "test": { + "sum": { + "field": "value" + } + } + } + } + ], + "filter_query": { + "bool": { + "filter": [ + { + "range": { + "value": { + "gt": 1 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "category_field": [ + "error_type" ] } } @@ -834,37 +1228,48 @@ POST _plugins/_anomaly_detection/detectors//_preview ```json { "anomaly_result": [ - ... { - "detector_id": "m4ccEnIBTXsGi3mvMt9p", - "data_start_time": 1588843020000, - "data_end_time": 1588843620000, + "detector_id": null, + "data_start_time": 1633049280000, + "data_end_time": 1633049340000, + "schema_version": 0, "feature_data": [ { - "feature_id": "xxokEnIBcpeWMD987A1X", - "feature_name": "total_order", - "data": 489.9929131106 + "feature_id": "tkTpTXwBjd8s6RK4DlOZ", + "feature_name": "test", + "data": 0 } ], "anomaly_grade": 0, - "confidence": 0.99 - } + "confidence": 0, + "entity": [ + { + "name": "error_type", + "value": "error1" + } + ] + }, ... ], "anomaly_detector": { "name": "test-detector", - "description": "Test detector", + "description": "Test update detector", "time_field": "timestamp", "indices": [ - "order*" + "server_log*" ], "filter_query": { "bool": { "filter": [ { - "exists": { - "field": "value", - "boost": 1 + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } } } ], @@ -874,24 +1279,25 @@ POST _plugins/_anomaly_detection/detectors//_preview }, "detection_interval": { "period": { - "interval": 10, - "unit": "MINUTES" + "interval": 1, + "unit": "Minutes" } }, "window_delay": { "period": { "interval": 1, - "unit": "MINUTES" + "unit": "Minutes" } }, + "shingle_size": 8, "schema_version": 0, "feature_attributes": [ { - "feature_id": "xxokEnIBcpeWMD987A1X", - "feature_name": "total_order", + "feature_id": "tkTpTXwBjd8s6RK4DlOZ", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "total_order": { + "test": { "sum": { "field": "value" } @@ -899,12 +1305,34 @@ POST _plugins/_anomaly_detection/detectors//_preview } } ], - "last_update_time": 1589442309241 + "category_field": [ + "error_type" + ], + "detector_type": "MULTI_ENTITY" } } ``` -If you specify a category field, each result is associated with an entity: +You can preview a detector with the detector ID: + +```json +POST _plugins/_anomaly_detection/detectors/_preview +{ + "detector_id": "VEHKTXwBwf_U8gjUXY2s", + "period_start": 1633048868000, + "period_end": 1633394468000 +} +``` + +Or: + +```json +POST _opendistro/_anomaly_detection/detectors/VEHKTXwBwf_U8gjUXY2s/_preview +{ + "period_start": 1633048868000, + "period_end": 1633394468000 +} +``` #### Sample response @@ -912,53 +1340,103 @@ If you specify a category field, each result is associated with an entity: { "anomaly_result": [ { - "detector_id": "4CIGoHUBTpMGN-4KzBQg", - "data_start_time": 1604277960000, - "data_end_time": 1604278020000, + "detector_id": "VEHKTXwBwf_U8gjUXY2s", + "data_start_time": 1633049280000, + "data_end_time": 1633049340000, "schema_version": 0, + "feature_data": [ + { + "feature_id": "3kHiTXwBwf_U8gjUlY15", + "feature_name": "test", + "data": 0 + } + ], "anomaly_grade": 0, - "confidence": 0.99 - } + "confidence": 0, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + } + }, + ... ], - "entity": [ - { - "name": "host", - "value": "i-00f28ec1eb8997686" - } - ] -}, -{ - "detector_id": "4CIGoHUBTpMGN-4KzBQg", - "data_start_time": 1604278020000, - "data_end_time": 1604278080000, - "schema_version": 0, - "feature_data": [ - { - "feature_id": "0Kld3HUBhpHMyt2e_UHn", - "feature_name": "latency_max", - "data": -17 - } - ], - "anomaly_grade": 0, - "confidence": 0.99, - "entity": [ - { - "name": "host", - "value": "i-00f28ec1eb8997686" - } - ] -} -... -``` - -Or, you can specify the detector ID: - -```json -POST _plugins/_anomaly_detection/detectors/_preview -{ - "detector_id": "sYkUvHcBiZv51f-Lv8QN", - "period_start": 1612982516000, - "period_end": 1614278539000 + "anomaly_detector": { + "name": "test-detector", + "description": "Test update detector", + "time_field": "timestamp", + "indices": [ + "server_log*" + ], + "filter_query": { + "bool": { + "filter": [ + { + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "3kHiTXwBwf_U8gjUlY15", + "feature_name": "test", + "feature_enabled": true, + "aggregation_query": { + "test": { + "sum": { + "field": "value" + } + } + } + } + ], + "last_update_time": 1633394267522, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "SINGLE_ENTITY" + } } ``` @@ -970,6 +1448,7 @@ Introduced 1.0 Starts a real-time or historical anomaly detector job. +To start a real-time detector job: #### Request @@ -981,23 +1460,38 @@ POST _plugins/_anomaly_detection/detectors//_start ```json { - "_id" : "m4ccEnIBTXsGi3mvMt9p", - "_version" : 1, - "_seq_no" : 6, - "_primary_term" : 1 + "_id": "VEHKTXwBwf_U8gjUXY2s", + "_version": 3, + "_seq_no": 6, + "_primary_term": 1 } ``` +The `_id` represents the real-time job ID, which is the same as the detector ID. + To start historical analysis: ```json POST _plugins/_anomaly_detection/detectors//_start { - "start_time": 1503168590000, - "end_time": 1617301324000 + "start_time": 1633048868000, + "end_time": 1633394468000 } ``` +#### Sample response + +```json +{ + "_id": "f9DsTXwB6HknB84SoRTY", + "_version": 1, + "_seq_no": 958, + "_primary_term": 1 +} +``` + +The `_id` represents the historical batch task ID, which is a random universally unique identifier (UUID). + --- ## Stop detector job @@ -1006,6 +1500,8 @@ Introduced 1.0 Stops a real-time or historical anomaly detector job. +To stop a real-time detector job: + #### Request ```json @@ -1015,7 +1511,12 @@ POST _plugins/_anomaly_detection/detectors//_stop #### Sample response ```json -Stopped detector: m4ccEnIBTXsGi3mvMt9p +{ + "_id": "VEHKTXwBwf_U8gjUXY2s", + "_version": 0, + "_seq_no": 0, + "_primary_term": 0 +} ``` To stop historical analysis: @@ -1024,30 +1525,37 @@ To stop historical analysis: POST _plugins/_anomaly_detection/detectors//_stop?historical=true ``` +#### Sample response + +```json +{ + "_id": "f9DsTXwB6HknB84SoRTY", + "_version": 0, + "_seq_no": 0, + "_primary_term": 0 +} +``` --- -## Search detector result +## Search detector Introduced 1.0 {: .label .label-purple } -Returns all results for a search query. +Returns all anomaly detectors for a search query. + +To search detectors using the `server_log*` index: #### Request ```json -GET _plugins/_anomaly_detection/detectors/results/_search -POST _plugins/_anomaly_detection/detectors/results/_search +GET _plugins/_anomaly_detection/detectors/_search +POST _plugins/_anomaly_detection/detectors/_search { "query": { - "bool": { - "must": { - "range": { - "anomaly_score": { - "gte": 0.6, - "lte": 1 - } - } + "wildcard": { + "indices": { + "value": "server_log*" } } } @@ -1058,138 +1566,7 @@ POST _plugins/_anomaly_detection/detectors/results/_search ```json { - "took": 9, - "timed_out": false, - "_shards": { - "total": 25, - "successful": 25, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": { - "value": 2, - "relation": "eq" - }, - "max_score": 1, - "hits": [ - { - "_index": ".opensearch-anomaly-results-history-2020.04.30-1", - "_type": "_doc", - "_id": "_KBrzXEBbpoKkFM5mStm", - "_version": 1, - "_seq_no": 58, - "_primary_term": 1, - "_score": 1, - "_source": { - "detector_id": "2KDozHEBbpoKkFM58yr6", - "anomaly_score": 0.8995068350366767, - "execution_start_time": 1588289313114, - "data_end_time": 1588289313114, - "confidence": 0.84214852704501, - "data_start_time": 1588289253114, - "feature_data": [ - { - "feature_id": "X0fpzHEB5NGZmIRkXKcy", - "feature_name": "total_error", - "data": 20 - } - ], - "execution_end_time": 1588289313126, - "anomaly_grade": 0 - } - }, - { - "_index": ".opensearch-anomaly-results-history-2020.04.30-1", - "_type": "_doc", - "_id": "EqB1zXEBbpoKkFM5qyyE", - "_version": 1, - "_seq_no": 61, - "_primary_term": 1, - "_score": 1, - "_source": { - "detector_id": "2KDozHEBbpoKkFM58yr6", - "anomaly_score": 0.7086834513354907, - "execution_start_time": 1588289973113, - "data_end_time": 1588289973113, - "confidence": 0.42162017029510446, - "data_start_time": 1588289913113, - "feature_data": [ - { - "feature_id": "X0fpzHEB5NGZmIRkXKcy", - "feature_name": "memory_usage", - "data": 20.0347333108 - } - ], - "execution_end_time": 1588289973124, - "anomaly_grade": 0 - } - } - ] - } -} -``` - -In high cardinality detectors, the result contains entity information. - -To see an ordered set of anomaly records for an entity with an anomaly within a certain time range for a specific feature value: - -#### Request - -```json -POST _plugins/_anomaly_detection/detectors/results/_search -{ - "query": { - "bool": { - "filter": [ - { - "term": { - "detector_id": "4CIGoHUBTpMGN-4KzBQg" - } - }, - { - "range": { - "anomaly_grade": { - "gt": 0 - } - } - }, - { - "nested": { - "path": "entity", - "query": { - "bool": { - "must": [ - { - "term": { - "entity.value": "i-00f28ec1eb8997685" - } - } - ] - } - } - } - } - ] - } - }, - "size": 8, - "sort": [ - { - "execution_end_time": { - "order": "desc" - } - } - ], - "track_total_hits": true -} -``` - -#### Sample response - -```json -{ - "took": 443, + "took": 2, "timed_out": false, "_shards": { "total": 1, @@ -1199,136 +1576,119 @@ POST _plugins/_anomaly_detection/detectors/results/_search }, "hits": { "total": { - "value": 7, + "value": 4, "relation": "eq" }, - "max_score": null, + "max_score": 1, "hits": [ { - "_index": ".opensearch-anomaly-results-history-2020.11.07-1", + "_index": ".opendistro-anomaly-detectors", "_type": "_doc", - "_id": "BiItoHUBTpMGN-4KARY5", + "_id": "Zi5zTXwBwf_U8gjUTfJG", "_version": 1, - "_seq_no": 206, + "_seq_no": 1, "_primary_term": 1, - "_score": null, + "_score": 1, "_source": { - "detector_id": "4CIGoHUBTpMGN-4KzBQg", - "schema_version": 2, - "anomaly_score": 2.462550517055763, - "execution_start_time": 1604710105400, - "data_end_time": 1604710094516, - "confidence": 0.8246254862573076, - "data_start_time": 1604710034516, - "feature_data": [ + "name": "test", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" + ], + "filter_query": { + "match_all": { + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 5, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ { - "feature_id": "0Kld3HUBhpHMyt2e_UHn", - "feature_name": "latency_max", - "data": 3526 + "feature_id": "ZS5zTXwBwf_U8gjUTfIn", + "feature_name": "test_feature", + "feature_enabled": true, + "aggregation_query": { + "test_feature": { + "sum": { + "field": "value" + } + } + } } ], - "execution_end_time": 1604710105401, - "anomaly_grade": 0.08045977011494891, - "entity": [ - { - "name": "host", - "value": "i-00f28ec1eb8997685" - } - ] - }, - "sort": [ - 1604710105401 - ] + "last_update_time": 1633386974533, + "category_field": [ + "error_type" + ], + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "MULTI_ENTITY" + } }, - { - "_index": ".opensearch-anomaly-results-history-2020.11.07-1", - "_type": "_doc", - "_id": "wiImoHUBTpMGN-4KlhXs", - "_version": 1, - "_seq_no": 156, - "_primary_term": 1, - "_score": null, - "_source": { - "detector_id": "4CIGoHUBTpMGN-4KzBQg", - "schema_version": 2, - "anomaly_score": 4.892453213261217, - "execution_start_time": 1604709684971, - "data_end_time": 1604709674522, - "confidence": 0.8313735633713821, - "data_start_time": 1604709614522, - "feature_data": [ - { - "feature_id": "0Kld3HUBhpHMyt2e_UHn", - "feature_name": "latency_max", - "data": 5709 - } - ], - "execution_end_time": 1604709684971, - "anomaly_grade": 0.06542056074767538, - "entity": [ - { - "name": "host", - "value": "i-00f28ec1eb8997685" - } - ] - }, - "sort": [ - 1604709684971 - ] - }, - { - "_index": ".opensearch-anomaly-results-history-2020.11.07-1", - "_type": "_doc", - "_id": "ZiIcoHUBTpMGN-4KhhVA", - "_version": 1, - "_seq_no": 79, - "_primary_term": 1, - "_score": null, - "_source": { - "detector_id": "4CIGoHUBTpMGN-4KzBQg", - "schema_version": 2, - "anomaly_score": 3.187717536855158, - "execution_start_time": 1604709025343, - "data_end_time": 1604709014520, - "confidence": 0.8301116064308817, - "data_start_time": 1604708954520, - "feature_data": [ - { - "feature_id": "0Kld3HUBhpHMyt2e_UHn", - "feature_name": "latency_max", - "data": 441 - } - ], - "execution_end_time": 1604709025344, - "anomaly_grade": 0.040767386091133916, - "entity": [ - { - "name": "host", - "value": "i-00f28ec1eb8997685" - } - ] - }, - "sort": [ - 1604709025344 - ] - } + ... ] } } ``` -You can query the anomaly results for historical analysis with the `task_id`: +--- + +## Search detector tasks +Introduced 1.1 +{: .label .label-purple } + +Searches detector tasks. + +To search for the latest detector task for a high cardinality detector: #### Request ```json -GET _plugins/_anomaly_detection/detectors/results/_search +GET _plugins/_anomaly_detection/detectors/tasks/_search +POST _plugins/_anomaly_detection/detectors/tasks/_search { "query": { - "term": { - "task_id": { - "value": "NnlV9HUBQxqfQ7vBJNzy" - } + "bool": { + "filter": [ + { + "term": { + "detector_id": "Zi5zTXwBwf_U8gjUTfJG" + } + }, + { + "term": { + "task_type": "HISTORICAL_HC_DETECTOR" + } + }, + { + "term": { + "is_latest": "true" + } + } + ] } } } @@ -1351,211 +1711,41 @@ GET _plugins/_anomaly_detection/detectors/results/_search "value": 1, "relation": "eq" }, - "max_score": 2.1366, + "max_score": 0, "hits": [ { - "_index": ".opensearch-anomaly-detection-state", + "_index": ".opendistro-anomaly-detection-state", "_type": "_doc", - "_id": "CoM8WncBtt2qvI-LZO7_", - "_version": 8, - "_seq_no": 1351, - "_primary_term": 3, - "_score": 2.1366, - "_source": { - "detector_id": "dZc8WncBgO2zoQoFWVBA", - "worker_node": "dk6-HuKQRMKm2fi8TSDHsg", - "task_progress": 0.09486946, - "last_update_time": 1612126667008, - "execution_start_time": 1612126643455, - "state": "RUNNING", - "coordinating_node": "gs213KqjS4q7H4Bmn_ZuLA", - "current_piece": 1583503800000, - "task_type": "HISTORICAL", - "started_by": "admin", - "init_progress": 1, - "is_latest": true, - "detector": { - "description": "test", - "ui_metadata": { - "features": { - "F1": { - "aggregationBy": "sum", - "aggregationOf": "value", - "featureType": "simple_aggs" - } - } - }, - "detection_date_range": { - "start_time": 1580504240308, - "end_time": 1612126640308 - }, - "feature_attributes": [ - { - "feature_id": "dJc8WncBgO2zoQoFWVAt", - "feature_enabled": true, - "feature_name": "F1", - "aggregation_query": { - "f_1": { - "sum": { - "field": "value" - } - } - } - } - ], - "schema_version": 0, - "time_field": "timestamp", - "last_update_time": 1612126640448, - "indices": [ - "server_log" - ], - "window_delay": { - "period": { - "unit": "Minutes", - "interval": 1 - } - }, - "detection_interval": { - "period": { - "unit": "Minutes", - "interval": 10 - } - }, - "name": "test-historical-detector", - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "shingle_size": 8, - "user": { - "backend_roles": [ - "admin" - ], - "custom_attribute_names": [], - "roles": [ - "all_access", - "own_index" - ], - "name": "admin", - "user_requested_tenant": "__user__" - }, - "detector_type": "HISTORICAL_SINGLE_ENTITY" - }, - "user": { - "backend_roles": [ - "admin" - ], - "custom_attribute_names": [], - "roles": [ - "all_access", - "own_index" - ], - "name": "admin", - "user_requested_tenant": "__user__" - } - } - } - ] - } -} -``` - ---- - -## Search detector tasks -Introduced 1.1 -{: .label .label-purple } - -Searches detector tasks. - -#### Request - -```json -POST _plugins/_anomaly_detection/detectors/tasks/_search -{ - "query": { - "bool": { - "filter": [ - { - "term": { - "detector_id": { - "value": "_6WPu3cBBnauGn7oxUAv" - } - } - }, - { - "term": { - "task_type": { - "value": "HISTORICAL_HC_DETECTOR" - } - } - } - ] - } - }, - "sort": [ - { - "execution_start_time": { - "order": "desc" - } - } - ] -} -``` - - -#### Sample response - -```json -{ - "took": 5, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": { - "value": 1, - "relation": "eq" - }, - "max_score": null, - "hits": [ - { - "_index": ".opensearch-anomaly-detection-state", - "_type": "_doc", - "_id": "TM3tOHwBCi2h__AOXlyQ", - "_version": 3, - "_seq_no": 14, + "_id": "fm-RTXwBYwCbWecgB753", + "_version": 34, + "_seq_no": 928, "_primary_term": 1, - "_score": null, + "_score": 0, "_source": { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "detector_id": "Zi5zTXwBwf_U8gjUTfJG", "error": "", "detection_date_range": { - "start_time": 1632437852100, - "end_time": 1633042652100 + "start_time": 1630794960000, + "end_time": 1633386960000 }, - "task_progress": 0.5, - "last_update_time": 1633042667358, - "execution_start_time": 1633042652810, - "state": "RUNNING", - "coordinating_node": "2hEGbUw6ShaiKe05n_xLdA", + "task_progress": 1, + "last_update_time": 1633389090738, + "execution_start_time": 1633388922742, + "state": "FINISHED", + "coordinating_node": "2Z4q22BySEyzakYt_A0A2A", "task_type": "HISTORICAL_HC_DETECTOR", + "execution_end_time": 1633389090738, "started_by": "admin", "init_progress": 0, "is_latest": true, "detector": { "category_field": [ - "type" + "error_type" ], "description": "test", "ui_metadata": { "features": { - "test-feature": { + "test_feature": { "aggregationBy": "sum", "aggregationOf": "value", "featureType": "simple_aggs" @@ -1565,9 +1755,9 @@ POST _plugins/_anomaly_detection/detectors/tasks/_search }, "feature_attributes": [ { - "feature_id": "7VDtOHwBD5tpxlbyWqPs", + "feature_id": "ZS5zTXwBwf_U8gjUTfIn", "feature_enabled": true, - "feature_name": "test-feature", + "feature_name": "test_feature", "aggregation_query": { "test_feature": { "sum": { @@ -1579,7 +1769,7 @@ POST _plugins/_anomaly_detection/detectors/tasks/_search ], "schema_version": 0, "time_field": "timestamp", - "last_update_time": 1633042652012, + "last_update_time": 1633386974533, "indices": [ "server_log" ], @@ -1595,7 +1785,7 @@ POST _plugins/_anomaly_detection/detectors/tasks/_search "interval": 5 } }, - "name": "test-detector", + "name": "testhc", "filter_query": { "match_all": { "boost": 1 @@ -1612,7 +1802,7 @@ POST _plugins/_anomaly_detection/detectors/tasks/_search "all_access" ], "name": "admin", - "user_requested_tenant": null + "user_requested_tenant": "__user__" }, "detector_type": "MULTI_ENTITY" }, @@ -1628,50 +1818,1060 @@ POST _plugins/_anomaly_detection/detectors/tasks/_search "name": "admin", "user_requested_tenant": "__user__" } - }, - "sort": [ - 1633042652810 - ] + } } ] } } ``` - ---- - -## Delete detector -Introduced 1.0 -{: .label .label-purple } - -Deletes a detector based on the `detector_id`. -To delete a detector, you need to first stop the detector. +To search for the latest entity-level task for a high cardinality detector: #### Request ```json -DELETE _plugins/_anomaly_detection/detectors/ +GET _plugins/_anomaly_detection/detectors/tasks/_search +POST _plugins/_anomaly_detection/detectors/tasks/_search +{ + "query": { + "bool": { + "filter": [ + { + "term": { + "detector_id": "Zi5zTXwBwf_U8gjUTfJG" + } + }, + { + "term": { + "task_type": "HISTORICAL_HC_ENTITY" + } + }, + { + "term": { + "is_latest": "true" + } + } + ] + } + }, + "sort": [ + { + "execution_start_time": { + "order": "desc" + } + } + ], + "size": 100 +} ``` +To search for all entity-level batch task stats: + +#### Request + +```json +GET _plugins/_anomaly_detection/detectors/tasks/_search +POST _plugins/_anomaly_detection/detectors/tasks/_search +{ + "size": 0, + "query": { + "bool": { + "filter": [ + { + "term": { + "detector_id": { + "value": "Zi5zTXwBwf_U8gjUTfJG", + "boost": 1 + } + } + }, + { + "term": { + "parent_task_id": { + "value": "fm-RTXwBYwCbWecgB753", + "boost": 1 + } + } + }, + { + "terms": { + "task_type": [ + "HISTORICAL_HC_ENTITY" + ], + "boost": 1 + } + } + ] + } + }, + "aggs": { + "test": { + "terms": { + "field": "state", + "size": 100 + } + } + } +} +``` #### Sample response ```json { - "_index": ".opensearch-anomaly-detectors", - "_type": "_doc", - "_id": "m4ccEnIBTXsGi3mvMt9p", - "_version": 2, - "result": "deleted", - "forced_refresh": true, + "took": 2, + "timed_out": false, "_shards": { - "total": 2, - "successful": 2, + "total": 1, + "successful": 1, + "skipped": 0, "failed": 0 }, - "_seq_no": 6, - "_primary_term": 1 + "hits": { + "total": { + "value": 32, + "relation": "eq" + }, + "max_score": null, + "hits": [] + }, + "aggregations": { + "test": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "FINISHED", + "doc_count": 32 + } + ] + } + } +} +``` + +--- + +## Search detector result +Introduced 1.0 +{: .label .label-purple } + +Returns all results for a search query. + +To search anomaly results for `grade` greater than 0 for real-time analysis: + +#### Request + +```json +GET _plugins/_anomaly_detection/detectors/results/_search +POST _plugins/_anomaly_detection/detectors/results/_search +{ + "query": { + "bool": { + "filter": [ + { + "term": { + "detector_id": "Zi5zTXwBwf_U8gjUTfJG" + } + }, + { + "range": { + "anomaly_grade": { + "gt": 0 + } + } + } + ], + "must_not": [ + { + "exists": { + "field": "task_id" + } + } + ] + } + } +} +``` + +Because real-time analysis doesn't have a batch task, the task ID in the anomaly result is null. + +#### Sample response + +```json +{ + "took": 4, + "timed_out": false, + "_shards": { + "total": 3, + "successful": 3, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 90, + "relation": "eq" + }, + "max_score": 0, + "hits": [ + { + "_index": ".opendistro-anomaly-results-history-2021.10.04-1", + "_type": "_doc", + "_id": "686KTXwB6HknB84SMr6G", + "_version": 1, + "_seq_no": 103622, + "_primary_term": 1, + "_score": 0, + "_source": { + "detector_id": "Zi5zTXwBwf_U8gjUTfJG", + "confidence": 0.918886275269358, + "model_id": "Zi5zTXwBwf_U8gjUTfJG_entity_error16", + "schema_version": 4, + "anomaly_score": 1.1093755891885446, + "execution_start_time": 1633388475001, + "data_end_time": 1633388414989, + "data_start_time": 1633388114989, + "feature_data": [ + { + "feature_id": "ZS5zTXwBwf_U8gjUTfIn", + "feature_name": "test_feature", + "data": 0.532 + } + ], + "execution_end_time": 1633388475014, + "user": { + "backend_roles": [ + "admin" + ], + "custom_attribute_names": [], + "roles": [ + "own_index", + "all_access" + ], + "name": "admin", + "user_requested_tenant": "__user__" + }, + "anomaly_grade": 0.031023547546561225, + "entity": [ + { + "name": "error_type", + "value": "error16" + } + ] + } + }, + ... + ] + } +} +``` + +You can run historical analysis as many times as you like. So, multiple tasks might exist for the same detector. + +You can search for the latest historical batch task first and then search the historical batch task results. + +To search anomaly results for `grade` greater than 0 for historical analysis with the `task_id`: + +#### Request + +```json +GET _plugins/_anomaly_detection/detectors/results/_search +POST _plugins/_anomaly_detection/detectors/results/_search +{ + "query": { + "bool": { + "filter": [ + { + "term": { + "detector_id": "Zi5zTXwBwf_U8gjUTfJG" + } + }, + { + "range": { + "anomaly_grade": { + "gt": 0 + } + } + }, + { + "term": { + "task_id": "fm-RTXwBYwCbWecgB753" + } + } + ] + } + } +} +``` + +#### Sample response + +```json +{ + "took": 915, + "timed_out": false, + "_shards": { + "total": 3, + "successful": 3, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 4115, + "relation": "eq" + }, + "max_score": 0, + "hits": [ + { + "_index": ".opendistro-anomaly-results-history-2021.10.04-1", + "_type": "_doc", + "_id": "VRyRTXwBDx7vzPBV8jYC", + "_version": 1, + "_seq_no": 149657, + "_primary_term": 1, + "_score": 0, + "_source": { + "detector_id": "Zi5zTXwBwf_U8gjUTfJG", + "confidence": 0.9642989263957601, + "task_id": "fm-RTXwBYwCbWecgB753", + "model_id": "Zi5zTXwBwf_U8gjUTfJG_entity_error24", + "schema_version": 4, + "anomaly_score": 1.2260712437521946, + "execution_start_time": 1633388982692, + "data_end_time": 1631721300000, + "data_start_time": 1631721000000, + "feature_data": [ + { + "feature_id": "ZS5zTXwBwf_U8gjUTfIn", + "feature_name": "test_feature", + "data": 10 + } + ], + "execution_end_time": 1633388982709, + "user": { + "backend_roles": [ + "admin" + ], + "custom_attribute_names": [], + "roles": [ + "own_index", + "all_access" + ], + "name": "admin", + "user_requested_tenant": "__user__" + }, + "anomaly_grade": 0.14249628345655782, + "entity": [ + { + "name": "error_type", + "value": "error1" + } + ] + } + }, + ... + ] + } +} +``` + +--- + +## Get detector stats +Introduced 1.0 +{: .label .label-purple } + +Provides information about how the plugin is performing. + +To get all stats: + +#### Request + +```json +GET _plugins/_anomaly_detection/stats +``` + +#### Sample response + +```json +{ + "anomaly_detectors_index_status": "green", + "anomaly_detection_state_status": "green", + "single_entity_detector_count": 2, + "detector_count": 5, + "multi_entity_detector_count": 3, + "anomaly_detection_job_index_status": "green", + "models_checkpoint_index_status": "green", + "anomaly_results_index_status": "green", + "nodes": { + "2Z4q22BySEyzakYt_A0A2A": { + "ad_execute_request_count": 95, + "models": [ + { + "detector_id": "WTBnTXwBjd8s6RK4b1Sz", + "model_type": "rcf", + "last_used_time": 1633398197185, + "model_id": "WTBnTXwBjd8s6RK4b1Sz_model_rcf_0", + "last_checkpoint_time": 1633396573679 + }, + ... + ], + "ad_canceled_batch_task_count": 0, + "ad_hc_execute_request_count": 75, + "ad_hc_execute_failure_count": 0, + "model_count": 28, + "ad_execute_failure_count": 1, + "ad_batch_task_failure_count": 0, + "ad_total_batch_task_execution_count": 27, + "ad_executing_batch_task_count": 3 + }, + "SWD7ihu9TaaW1zKwFZNVNg": { + "ad_execute_request_count": 12, + "models": [ + { + "detector_id": "Zi5zTXwBwf_U8gjUTfJG", + "model_type": "entity", + "last_used_time": 1633398375008, + "model_id": "Zi5zTXwBwf_U8gjUTfJG_entity_error13", + "last_checkpoint_time": 1633392973682, + "entity": [ + { + "name": "error_type", + "value": "error13" + } + ] + }, + ... + ], + "ad_canceled_batch_task_count": 1, + "ad_hc_execute_request_count": 0, + "ad_hc_execute_failure_count": 0, + "model_count": 15, + "ad_execute_failure_count": 2, + "ad_batch_task_failure_count": 0, + "ad_total_batch_task_execution_count": 27, + "ad_executing_batch_task_count": 4 + }, + "TQDUXEzyTJyV0H6_T4hYUw": { + "ad_execute_request_count": 0, + "models": [ + { + "detector_id": "Zi5zTXwBwf_U8gjUTfJG", + "model_type": "entity", + "last_used_time": 1633398375004, + "model_id": "Zi5zTXwBwf_U8gjUTfJG_entity_error24", + "last_checkpoint_time": 1633388177359, + "entity": [ + { + "name": "error_type", + "value": "error24" + } + ] + }, + ... + ], + "ad_canceled_batch_task_count": 0, + "ad_hc_execute_request_count": 0, + "ad_hc_execute_failure_count": 0, + "model_count": 22, + "ad_execute_failure_count": 0, + "ad_batch_task_failure_count": 0, + "ad_total_batch_task_execution_count": 28, + "ad_executing_batch_task_count": 3 + } + } +} +``` + +The `model_count` parameter shows the total number of models running on each node’s memory. +For historical analysis, you see the values for the following fields: + +- `ad_total_batch_task_execution_count` +- `ad_executing_batch_task_count` +- `ad_canceled_batch_task_count` +- `ad_batch_task_failure_count` + +If haven't run any historical analysis, these values show up as 0. + +To get all stats for a specific node: + +#### Request + +```json +GET _plugins/_anomaly_detection//stats +``` + +To get specific stats for a node: + +#### Request + +```json +GET _plugins/_anomaly_detection//stats/ +GET _plugins/_anomaly_detection//stats/ad_execute_request_count +``` + +#### Sample response + +```json +{ + "nodes": { + "SWD7ihu9TaaW1zKwFZNVNg": { + "ad_execute_request_count": 12 + } + } +} +``` + +To get a specific type of stats: + +#### Request + +```json +GET _plugins/_anomaly_detection/stats/ +GET _plugins/_anomaly_detection/stats/ad_executing_batch_task_count +``` + +#### Sample response + +```json +{ + "nodes": { + "2Z4q22BySEyzakYt_A0A2A": { + "ad_executing_batch_task_count": 3 + }, + "SWD7ihu9TaaW1zKwFZNVNg": { + "ad_executing_batch_task_count": 3 + }, + "TQDUXEzyTJyV0H6_T4hYUw": { + "ad_executing_batch_task_count": 4 + } + } +} +``` + +--- + +## Profile detector +Introduced 1.0 +{: .label .label-purple } + +Returns information related to the current state of the detector and memory usage, including current errors and shingle size, to help troubleshoot the detector. + +This command helps locate logs by identifying the nodes that run the anomaly detector job for each detector. + +It also helps track the initialization percentage, the required shingles, and the estimated time left. + +#### Request + +```json +GET _plugins/_anomaly_detection/detectors//_profile/ +GET _plugins/_anomaly_detection/detectors//_profile?_all=true +GET _plugins/_anomaly_detection/detectors//_profile/ +GET _plugins/_anomaly_detection/detectors//_profile/, +``` + +#### Sample Responses + +```json +GET _plugins/_anomaly_detection/detectors//_profile + +{ + "state": "DISABLED", + "error": "Stopped detector: AD models memory usage exceeds our limit." +} + +GET _plugins/_anomaly_detection/detectors//_profile?_all=true&pretty + +{ + "state": "RUNNING", + "error": "", + "models": [ + { + "model_id": "3Dh6TXwBwf_U8gjURE0F_entity_KSLSh0Wv05RQXiBAQHTEZg", + "entity": [ + { + "name": "ip", + "value": "192.168.1.1" + }, + { + "name": "error_type", + "value": "error8" + } + ], + "model_size_in_bytes": 403491, + "node_id": "2Z4q22BySEyzakYt_A0A2A" + }, + ... + ], + "total_size_in_bytes": 12911712, + "init_progress": { + "percentage": "100%" + }, + "total_entities": 33, + "active_entities": 32, + "ad_task": { + "ad_task": { + "task_id": "D3I5TnwBYwCbWecg7lN9", + "last_update_time": 1633399993685, + "started_by": "admin", + "state": "RUNNING", + "detector_id": "3Dh6TXwBwf_U8gjURE0F", + "task_progress": 0, + "init_progress": 0, + "execution_start_time": 1633399991933, + "is_latest": true, + "task_type": "HISTORICAL_HC_DETECTOR", + "coordinating_node": "2Z4q22BySEyzakYt_A0A2A", + "detector": { + "name": "testhc-mc", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" + ], + "filter_query": { + "match_all": { + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 5, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "2zh6TXwBwf_U8gjUQ039", + "feature_name": "test", + "feature_enabled": true, + "aggregation_query": { + "test": { + "sum": { + "field": "value" + } + } + } + } + ], + "ui_metadata": { + "features": { + "test": { + "aggregationBy": "sum", + "aggregationOf": "value", + "featureType": "simple_aggs" + } + }, + "filters": [] + }, + "last_update_time": 1633387430916, + "category_field": [ + "ip", + "error_type" + ], + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "MULTI_ENTITY" + }, + "detection_date_range": { + "start_time": 1632793800000, + "end_time": 1633398600000 + }, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + } + }, + "node_id": "2Z4q22BySEyzakYt_A0A2A", + "task_id": "D3I5TnwBYwCbWecg7lN9", + "task_type": "HISTORICAL_HC_DETECTOR", + "detector_task_slots": 10, + "total_entities_count": 32, + "pending_entities_count": 22, + "running_entities_count": 10, + "running_entities": [ """[{"name":"ip","value":"192.168.1.1"},{"name":"error_type","value":"error9"}]""", + ...], + "entity_task_profiles": [ + { + "shingle_size": 8, + "rcf_total_updates": 1994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "2Z4q22BySEyzakYt_A0A2A", + "entity": [ + { + "name": "ip", + "value": "192.168.1.1" + }, + { + "name": "error_type", + "value": "error7" + } + ], + "task_id": "E3I5TnwBYwCbWecg9FMm", + "task_type": "HISTORICAL_HC_ENTITY" + }, + ... + ] + }, + "model_count": 32 +} + +GET _plugins/_anomaly_detection/detectors//_profile/total_size_in_bytes + +{ + "total_size_in_bytes": 13369344 +} +``` + +You can see the `ad_task` field only for historical analysis. + +The `model_count` parameter shows the total number of models that a detector runs on each node’s memory. This is useful if you have several models running on your cluster and want to know the count. + +If you configured the category field, you can see the number of unique values in the field and all active entities with models running in memory. + +You can use this data to estimate how much memory is required for anomaly detection so you can decide how to size your cluster. For example, if a detector has one million entities and only 10 of them are active in memory, you need to scale your cluster up or out. + +For a single-entity detector: + +#### Sample response + +```json +{ + "state": "INIT", + "total_size_in_bytes": 0, + "init_progress": { + "percentage": "0%", + "needed_shingles": 128 + }, + "ad_task": { + "ad_task": { + "task_id": "cfUNOXwBFLNqSEcxAlde", + "last_update_time": 1633044731640, + "started_by": "admin", + "state": "RUNNING", + "detector_id": "qL4NOXwB__6eNorTAKtJ", + "task_progress": 0.49603173, + "init_progress": 1, + "current_piece": 1632739800000, + "execution_start_time": 1633044726365, + "is_latest": true, + "task_type": "HISTORICAL_SINGLE_ENTITY", + "coordinating_node": "bCtWtxWPThq0BIn5P5I4Xw", + "worker_node": "dIyavWhmSYWGz65b4u-lpQ", + "detector": { + "name": "detector1", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" + ], + "filter_query": { + "match_all": { + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 5, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "p74NOXwB__6eNorTAKss", + "feature_name": "test-feature", + "feature_enabled": true, + "aggregation_query": { + "test_feature": { + "sum": { + "field": "value" + } + } + } + } + ], + "ui_metadata": { + "features": { + "test-feature": { + "aggregationBy": "sum", + "aggregationOf": "value", + "featureType": "simple_aggs" + } + }, + "filters": [] + }, + "last_update_time": 1633044725832, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "SINGLE_ENTITY" + }, + "detection_date_range": { + "start_time": 1632439925885, + "end_time": 1633044725885 + }, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + } + }, + "shingle_size": 8, + "rcf_total_updates": 1994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "dIyavWhmSYWGz65b4u-lpQ", + "detector_task_slots": 1 + } +} +``` + +The `total_entities` parameter shows you the total number of entities including the number of category fields for a detector. + +Getting the total count of entities is an expensive operation for real-time analysis of a detector with more than one category field. By default, for a real-time detection profile, a detector counts the number of entities up to a value of 10,000. For historical analysis, the anomaly detection plugin only detects the top 1,000 entities by default and caches the top entities in memory, so it doesn't cost much to get the total count of entities for historical analysis. + +The `profile` operation also provides information about each entity, such as the entity’s `last_sample_timestamp` and `last_active_timestamp`. `last_sample_timestamp` shows the last document in the input data source index containing the entity, while `last_active_timestamp` shows the timestamp when the entity’s model was last seen in the model cache. + +If there are no anomaly results for an entity, either the entity doesn't have any sample data or its model is removed from the model cache. + +#### Request + +```json +GET _plugins/_anomaly_detection/detectors//_profile?_all=true +{ + "entity": [ + { + "name": "host", + "value": "i-00f28ec1eb8997686" + } + ] +} +``` + +#### Sample Responses + +```json +{ + "category_field": "host", + "value": "i-00f28ec1eb8997686", + "is_active": true, + "last_active_timestamp": 1604026394879, + "last_sample_timestamp": 1604026394879, + "init_progress": { + "percentage": "100%" + }, + "model": { + "model_id": "TFUdd3UBBwIAGQeRh5IS_entity_i-00f28ec1eb8997686", + "model_size_in_bytes": 712480, + "node_id": "MQ-bTBW3Q2uU_2zX3pyEQg" + }, + "state": "RUNNING" +} +``` + +To get profile information for only historical analysis, specify `ad_task`. +Specifying `_all` is an expensive operation for multi-category high cardinality detectors. + +#### Request + +```json +GET _plugins/_anomaly_detection/detectors//_profile?_all +GET _plugins/_anomaly_detection/detectors//_profile/ad_task +``` + +#### Sample Responses + +```json +{ + "ad_task": { + "ad_task": { + "task_id": "CHI0TnwBYwCbWecgqgRA", + "last_update_time": 1633399648413, + "started_by": "admin", + "state": "RUNNING", + "detector_id": "3Dh6TXwBwf_U8gjURE0F", + "task_progress": 0, + "init_progress": 0, + "execution_start_time": 1633399646784, + "is_latest": true, + "task_type": "HISTORICAL_HC_DETECTOR", + "coordinating_node": "2Z4q22BySEyzakYt_A0A2A", + "detector": { + "name": "testhc-mc", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" + ], + "filter_query": { + "match_all": { + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 5, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "2zh6TXwBwf_U8gjUQ039", + "feature_name": "test", + "feature_enabled": true, + "aggregation_query": { + "test": { + "sum": { + "field": "value" + } + } + } + } + ], + "ui_metadata": { + "features": { + "test": { + "aggregationBy": "sum", + "aggregationOf": "value", + "featureType": "simple_aggs" + } + }, + "filters": [] + }, + "last_update_time": 1633387430916, + "category_field": [ + "ip", + "error_type" + ], + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "MULTI_ENTITY" + }, + "detection_date_range": { + "start_time": 1632793800000, + "end_time": 1633398600000 + }, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + } + }, + "node_id": "2Z4q22BySEyzakYt_A0A2A", + "task_id": "CHI0TnwBYwCbWecgqgRA", + "task_type": "HISTORICAL_HC_DETECTOR", + "detector_task_slots": 10, + "total_entities_count": 32, + "pending_entities_count": 22, + "running_entities_count": 10, + "running_entities" : [ + """[{"name":"ip","value":"192.168.1.1"},{"name":"error_type","value":"error9"}]""", + ... + ], + "entity_task_profiles": [ + { + "shingle_size": 8, + "rcf_total_updates": 994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "2Z4q22BySEyzakYt_A0A2A", + "entity": [ + { + "name": "ip", + "value": "192.168.1.1" + }, + { + "name": "error_type", + "value": "error6" + } + ], + "task_id": "9XI0TnwBYwCbWecgsAd6", + "task_type": "HISTORICAL_HC_ENTITY" + }, + ... + ] + } } ``` @@ -1718,7 +2918,6 @@ DELETE _plugins/_anomaly_detection/detectors/results } ``` - #### Sample response ```json @@ -1743,486 +2942,6 @@ DELETE _plugins/_anomaly_detection/detectors/results } ``` - ---- - -## Search detector -Introduced 1.0 -{: .label .label-purple } - -Returns all anomaly detectors for a search query. - -#### Request - -```json -GET _plugins/_anomaly_detection/detectors/_search -POST _plugins/_anomaly_detection/detectors/_search -{ - "query": { - "bool": { - "filter": [ - { - "terms": { - "indices": [ - "server_log" - ] - } - } - ] - } - } -} -``` - -#### Sample response - -```json -{ - "took": 2, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": { - "value": 1, - "relation": "eq" - }, - "max_score": 0, - "hits": [ - { - "_index": ".opensearch-anomaly-detectors", - "_type": "_doc", - "_id": "rlDtOHwBD5tpxlbyW7Nt", - "_version": 1, - "_seq_no": 0, - "_primary_term": 1, - "_score": 0, - "_source": { - "name": "test-detector", - "description": "test", - "time_field": "timestamp", - "indices": [ - "server_log" - ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 5, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "7VDtOHwBD5tpxlbyWqPs", - "feature_name": "test-feature", - "feature_enabled": true, - "aggregation_query": { - "test_feature": { - "sum": { - "field": "value" - } - } - } - } - ], - "last_update_time": 1633042652012, - "category_field": [ - "type" - ], - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "own_index", - "all_access" - ], - "custom_attribute_names": [], - "user_requested_tenant": null - }, - "detector_type": "MULTI_ENTITY" - } - } - ] - } -} -``` - ---- - -## Get detector stats -Introduced 1.0 -{: .label .label-purple } - -Provides information about how the plugin is performing. - -#### Request - -```json -GET _plugins/_anomaly_detection/stats -GET _plugins/_anomaly_detection//stats -GET _plugins/_anomaly_detection//stats/ -GET _plugins/_anomaly_detection/stats/ -``` - -#### Sample response - -```json -{ - "anomaly_detectors_index_status": "yellow", - "anomaly_detection_state_status": "yellow", - "single_entity_detector_count": 0, - "detector_count": 1, - "multi_entity_detector_count": 0, - "anomaly_detection_job_index_status": "yellow", - "models_checkpoint_index_status": "yellow", - "anomaly_results_index_status": "yellow", - "nodes": { - "hhfW2ZNVTJCtbs8rO-nF4g": { - "ad_execute_request_count": 6, - "models": [ - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578975, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_ic43SHH11NWr32xXgjRFwg", - "last_checkpoint_time": 1629827339211, - "entity": [ - { - "name": "host", - "value": "server_2" - }, - { - "name": "service", - "value": "app_6" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578979, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_OIsxy2nXMVdngK6Vv3X0uw", - "last_checkpoint_time": 1629827342444, - "entity": [ - { - "name": "host", - "value": "server_2" - }, - { - "name": "service", - "value": "app_2" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578978, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_I0L8K8ktyVnyL59CVFCLVQ", - "last_checkpoint_time": 1629827342068, - "entity": [ - { - "name": "host", - "value": "server_1" - }, - { - "name": "service", - "value": "app_4" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578975, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_ZoNYVJsq5ry6e-SWXmAt1Q", - "last_checkpoint_time": 1629827339435, - "entity": [ - { - "name": "host", - "value": "server_1" - }, - { - "name": "service", - "value": "app_6" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578978, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_qo2ANH_NS7Bg8iV4AJpHOw", - "last_checkpoint_time": 1629827341187, - "entity": [ - { - "name": "host", - "value": "server_3" - }, - { - "name": "service", - "value": "app_0" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578980, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_412FoQwCykWTAhjVfDGQDg", - "last_checkpoint_time": 1629827342983, - "entity": [ - { - "name": "host", - "value": "server_1" - }, - { - "name": "service", - "value": "app_2" - } - ] - } - ], - "ad_canceled_batch_task_count": 0, - "ad_hc_execute_request_count": 6, - "ad_hc_execute_failure_count": 0, - "model_count": 21, - "ad_execute_failure_count": 0, - "ad_batch_task_failure_count": 0, - "ad_total_batch_task_execution_count": 0, - "ad_executing_batch_task_count": 0 - } - } -} -``` - -The `model_count` parameter shows the total number of models running on each node’s memory. -For historical analysis, you see the values for the following fields: - -- `ad_total_batch_task_execution_count` -- `ad_executing_batch_task_count` -- `ad_canceled_batch_task_count` -- `ad_batch_task_failure_count` - -If haven't run any historical analysis, these values show up as 0. - -#### Sample response - -```json -{ - "anomaly_detectors_index_status": "green", - "anomaly_detection_state_status": "green", - "single_entity_detector_count": 0, - "detector_count": 1, - "multi_entity_detector_count": 1, - "anomaly_detection_job_index_status": "green", - "models_checkpoint_index_status": "green", - "anomaly_results_index_status": "green", - "nodes": { - "bCtWtxWPThq0BIn5P5I4Xw": { - "ad_execute_request_count": 0, - "models": [ - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152729, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error20", - "last_checkpoint_time": 1633043556222, - "entity": [ - { - "name": "type", - "value": "error20" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152767, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error31", - "last_checkpoint_time": 1633043855146, - "entity": [ - { - "name": "type", - "value": "error31" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152732, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error16", - "last_checkpoint_time": 1633043557253, - "entity": [ - { - "name": "type", - "value": "error16" - } - ] - } - ], - "ad_canceled_batch_task_count": 0, - "ad_hc_execute_request_count": 0, - "ad_hc_execute_failure_count": 0, - "model_count": 8, - "ad_execute_failure_count": 0, - "ad_batch_task_failure_count": 0, - "ad_total_batch_task_execution_count": 15, - "ad_executing_batch_task_count": 3 - }, - "dIyavWhmSYWGz65b4u-lpQ": { - "ad_execute_request_count": 0, - "models": [ - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152729, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error3", - "last_checkpoint_time": 1633043256013, - "entity": [ - { - "name": "type", - "value": "error3" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152727, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error1", - "last_checkpoint_time": 1633043254819, - "entity": [ - { - "name": "type", - "value": "error1" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152735, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error22", - "last_checkpoint_time": 1633043557023, - "entity": [ - { - "name": "type", - "value": "error22" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152763, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error29", - "last_checkpoint_time": 1633043855226, - "entity": [ - { - "name": "type", - "value": "error29" - } - ] - } - ], - "ad_canceled_batch_task_count": 0, - "ad_hc_execute_request_count": 0, - "ad_hc_execute_failure_count": 0, - "model_count": 11, - "ad_execute_failure_count": 0, - "ad_batch_task_failure_count": 0, - "ad_total_batch_task_execution_count": 14, - "ad_executing_batch_task_count": 3 - }, - "2hEGbUw6ShaiKe05n_xLdA": { - "ad_execute_request_count": 5, - "models": [ - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152714, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error5", - "last_checkpoint_time": 1633043256689, - "entity": [ - { - "name": "type", - "value": "error5" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152720, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error15", - "last_checkpoint_time": 1633043553786, - "entity": [ - { - "name": "type", - "value": "error15" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152724, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error17", - "last_checkpoint_time": 1633043554909, - "entity": [ - { - "name": "type", - "value": "error17" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152751, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error27", - "last_checkpoint_time": 1633043855105, - "entity": [ - { - "name": "type", - "value": "error27" - } - ] - } - ], - "ad_canceled_batch_task_count": 0, - "ad_hc_execute_request_count": 5, - "ad_hc_execute_failure_count": 0, - "model_count": 13, - "ad_execute_failure_count": 0, - "ad_batch_task_failure_count": 0, - "ad_total_batch_task_execution_count": 14, - "ad_executing_batch_task_count": 3 - } - } -} -``` - --- ## Create monitor @@ -2444,899 +3163,3 @@ POST _plugins/_alerting/monitors ``` --- - -## Profile detector -Introduced 1.0 -{: .label .label-purple } - -Returns information related to the current state of the detector and memory usage, including current errors and shingle size, to help troubleshoot the detector. - -This command helps locate logs by identifying the nodes that run the anomaly detector job for each detector. - -It also helps track the initialization percentage, the required shingles, and the estimated time left. - -#### Request - -```json -GET _plugins/_anomaly_detection/detectors//_profile/ -GET _plugins/_anomaly_detection/detectors//_profile?_all=true -GET _plugins/_anomaly_detection/detectors//_profile/ -GET _plugins/_anomaly_detection/detectors//_profile/, -``` - -#### Sample Responses - -```json -GET _plugins/_anomaly_detection/detectors//_profile -{ - "state": "DISABLED", - "error": "Stopped detector: AD models memory usage exceeds our limit." -} - -GET _plugins/_anomaly_detection/detectors//_profile?_all=true&pretty -{ - "state": "RUNNING", - "error": "", - "models": [ - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error20", - "entity": [ - { - "name": "type", - "value": "error20" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error31", - "entity": [ - { - "name": "type", - "value": "error31" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error21", - "entity": [ - { - "name": "type", - "value": "error21" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error13", - "entity": [ - { - "name": "type", - "value": "error13" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error24", - "entity": [ - { - "name": "type", - "value": "error24" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error25", - "entity": [ - { - "name": "type", - "value": "error25" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error28", - "entity": [ - { - "name": "type", - "value": "error28" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error16", - "entity": [ - { - "name": "type", - "value": "error16" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error3", - "entity": [ - { - "name": "type", - "value": "error3" - } - ], - "model_size_in_bytes": 403491, - "node_id": "dIyavWhmSYWGz65b4u-lpQ" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error27", - "entity": [ - { - "name": "type", - "value": "error27" - } - ], - "model_size_in_bytes": 403491, - "node_id": "2hEGbUw6ShaiKe05n_xLdA" - } - ], - "total_size_in_bytes": 12911712, - "init_progress": { - "percentage": "100%" - }, - "total_entities": 33, - "active_entities": 32, - "ad_task": { - "ad_task": { - "task_id": "Os4HOXwBCi2h__AONgpc", - "last_update_time": 1633044347855, - "started_by": "admin", - "state": "RUNNING", - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "task_progress": 0, - "init_progress": 0, - "execution_start_time": 1633044346460, - "is_latest": true, - "task_type": "HISTORICAL_HC_DETECTOR", - "coordinating_node": "2hEGbUw6ShaiKe05n_xLdA", - "detector": { - "name": "test-detector", - "description": "test", - "time_field": "timestamp", - "indices": [ - "server_log" - ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 5, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "7VDtOHwBD5tpxlbyWqPs", - "feature_name": "test-feature", - "feature_enabled": true, - "aggregation_query": { - "test_feature": { - "sum": { - "field": "value" - } - } - } - } - ], - "ui_metadata": { - "features": { - "test-feature": { - "aggregationBy": "sum", - "aggregationOf": "value", - "featureType": "simple_aggs" - } - }, - "filters": [] - }, - "last_update_time": 1633042652012, - "category_field": [ - "type" - ], - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "own_index", - "all_access" - ], - "custom_attribute_names": [], - "user_requested_tenant": null - }, - "detector_type": "MULTI_ENTITY" - }, - "detection_date_range": { - "start_time": 1632437820000, - "end_time": 1633042620000 - }, - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "own_index", - "all_access" - ], - "custom_attribute_names": [], - "user_requested_tenant": "__user__" - } - }, - "node_id": "2hEGbUw6ShaiKe05n_xLdA", - "task_id": "Os4HOXwBCi2h__AONgpc", - "task_type": "HISTORICAL_HC_DETECTOR", - "detector_task_slots": 10, - "total_entities_count": 32, - "pending_entities_count": 22, - "running_entities_count": 10, - "running_entities": [ - "error9", - "error8", - "error7", - "error6", - "error5", - "error4", - "error32", - "error31", - "error30", - "error3" - ], - "entity_task_profiles": [ - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw", - "entity": [ - { - "name": "type", - "value": "error6" - } - ], - "task_id": "P84HOXwBCi2h__AOOgrC", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "dIyavWhmSYWGz65b4u-lpQ", - "entity": [ - { - "name": "type", - "value": "error4" - } - ], - "task_id": "Kc4HOXwBCi2h__AOOw6Y", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "dIyavWhmSYWGz65b4u-lpQ", - "entity": [ - { - "name": "type", - "value": "error8" - } - ], - "task_id": "Pc4HOXwBCi2h__AOOgqJ", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "2hEGbUw6ShaiKe05n_xLdA", - "entity": [ - { - "name": "type", - "value": "error7" - } - ], - "task_id": "Ps4HOXwBCi2h__AOOgqh", - "task_type": "HISTORICAL_HC_ENTITY" - } - ] - }, - "model_count": 32 -} - -GET _plugins/_anomaly_detection/detectors//_profile/total_size_in_bytes -{ - "total_size_in_bytes": 13369344 -} -``` - -You can see the `ad_task` field only for historical analysis. - -The `model_count` parameter shows the total number of models that a detector runs on each node’s memory. This is useful if you have several models running on your cluster and want to know the count. - -If you configured the category field, you can see the number of unique values in the field and all active entities with models running in memory. - -You can use this data to estimate how much memory is required for anomaly detection so you can decide how to size your cluster. For example, if a detector has one million entities and only 10 of them are active in memory, you need to scale your cluster up or out. - -#### Request - -```json -GET _plugins/_anomaly_detection/detectors//_profile?_all=true&pretty -{ - "state": "RUNNING", - "error": "", - "models": [ - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error20", - "entity": [ - { - "name": "type", - "value": "error20" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error13", - "entity": [ - { - "name": "type", - "value": "error13" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error27", - "entity": [ - { - "name": "type", - "value": "error27" - } - ], - "model_size_in_bytes": 403491, - "node_id": "2hEGbUw6ShaiKe05n_xLdA" - } - ], - "total_size_in_bytes": 12911712, - "init_progress": { - "percentage": "100%" - }, - "total_entities": 33, - "active_entities": 32, - "ad_task": { - "ad_task": { - "task_id": "Os4HOXwBCi2h__AONgpc", - "last_update_time": 1633044347855, - "started_by": "admin", - "state": "RUNNING", - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "task_progress": 0, - "init_progress": 0, - "execution_start_time": 1633044346460, - "is_latest": true, - "task_type": "HISTORICAL_HC_DETECTOR", - "coordinating_node": "2hEGbUw6ShaiKe05n_xLdA", - "detector": { - "name": "test-detector", - "description": "test", - "time_field": "timestamp", - "indices": [ - "server_log" - ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 5, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "7VDtOHwBD5tpxlbyWqPs", - "feature_name": "test-feature", - "feature_enabled": true, - "aggregation_query": { - "test_feature": { - "sum": { - "field": "value" - } - } - } - } - ], - "ui_metadata": { - "features": { - "test-feature": { - "aggregationBy": "sum", - "aggregationOf": "value", - "featureType": "simple_aggs" - } - }, - "filters": [] - }, - "last_update_time": 1633042652012, - "category_field": [ - "type" - ], - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "own_index", - "all_access" - ], - "custom_attribute_names": [], - "user_requested_tenant": null - }, - "detector_type": "MULTI_ENTITY" - }, - "detection_date_range": { - "start_time": 1632437820000, - "end_time": 1633042620000 - }, - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "own_index", - "all_access" - ], - "custom_attribute_names": [], - "user_requested_tenant": "__user__" - } - }, - "node_id": "2hEGbUw6ShaiKe05n_xLdA", - "task_id": "Os4HOXwBCi2h__AONgpc", - "task_type": "HISTORICAL_HC_DETECTOR", - "detector_task_slots": 10, - "total_entities_count": 32, - "pending_entities_count": 22, - "running_entities_count": 10, - "running_entities": [ - "error9", - "error8", - "error7", - "error6", - "error5", - "error4", - "error32", - "error31", - "error30", - "error3" - ], - "entity_task_profiles": [ - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw", - "entity": [ - { - "name": "type", - "value": "error6" - } - ], - "task_id": "P84HOXwBCi2h__AOOgrC", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw", - "entity": [ - { - "name": "type", - "value": "error5" - } - ], - "task_id": "QM4HOXwBCi2h__AOOgre", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw", - "entity": [ - { - "name": "type", - "value": "error9" - } - ], - "task_id": "PM4HOXwBCi2h__AOOgp3", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "2hEGbUw6ShaiKe05n_xLdA", - "entity": [ - { - "name": "type", - "value": "error3" - } - ], - "task_id": "Fs4HOXwBCi2h__AOPBLH", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "2hEGbUw6ShaiKe05n_xLdA", - "entity": [ - { - "name": "type", - "value": "error32" - } - ], - "task_id": "Ks4HOXwBCi2h__AOOw7D", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "2hEGbUw6ShaiKe05n_xLdA", - "entity": [ - { - "name": "type", - "value": "error7" - } - ], - "task_id": "Ps4HOXwBCi2h__AOOgqh", - "task_type": "HISTORICAL_HC_ENTITY" - } - ] - }, - "model_count": 32 -} -``` - -For a single-entity detector: - -#### Sample response - -```json -{ - "state": "INIT", - "total_size_in_bytes": 0, - "init_progress": { - "percentage": "0%", - "needed_shingles": 128 - }, - "ad_task": { - "ad_task": { - "task_id": "cfUNOXwBFLNqSEcxAlde", - "last_update_time": 1633044731640, - "started_by": "admin", - "state": "RUNNING", - "detector_id": "qL4NOXwB__6eNorTAKtJ", - "task_progress": 0.49603173, - "init_progress": 1, - "current_piece": 1632739800000, - "execution_start_time": 1633044726365, - "is_latest": true, - "task_type": "HISTORICAL_SINGLE_ENTITY", - "coordinating_node": "bCtWtxWPThq0BIn5P5I4Xw", - "worker_node": "dIyavWhmSYWGz65b4u-lpQ", - "detector": { - "name": "detector1", - "description": "test", - "time_field": "timestamp", - "indices": [ - "server_log" - ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 5, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "p74NOXwB__6eNorTAKss", - "feature_name": "test-feature", - "feature_enabled": true, - "aggregation_query": { - "test_feature": { - "sum": { - "field": "value" - } - } - } - } - ], - "ui_metadata": { - "features": { - "test-feature": { - "aggregationBy": "sum", - "aggregationOf": "value", - "featureType": "simple_aggs" - } - }, - "filters": [] - }, - "last_update_time": 1633044725832, - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "own_index", - "all_access" - ], - "custom_attribute_names": [], - "user_requested_tenant": "__user__" - }, - "detector_type": "SINGLE_ENTITY" - }, - "detection_date_range": { - "start_time": 1632439925885, - "end_time": 1633044725885 - }, - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "own_index", - "all_access" - ], - "custom_attribute_names": [], - "user_requested_tenant": "__user__" - } - }, - "shingle_size": 8, - "rcf_total_updates": 1994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "dIyavWhmSYWGz65b4u-lpQ", - "detector_task_slots": 1 - } -} -``` - -The `total_entities` parameter shows you the total number of entities including the number of category fields for a detector. - -Getting the total count of entities is an expensive operation for real-time analysis of a detector with more than one category field. By default, for a real-time detection profile, a detector counts the number of entities up to a value of 10,000. For historical analysis, the anomaly detection plugin only detects the top 1,000 entities by default and caches the top entities in memory to reduce the cost of getting the total count of entities for historical analysis. - -The `profile` operation also provides information about each entity, such as the entity’s `last_sample_timestamp` and `last_active_timestamp`. `last_sample_timestamp` shows the last document in the input data source index containing the entity, while `last_active_timestamp` shows the timestamp when the entity’s model was last seen in the model cache. - -If there are no anomaly results for an entity, either the entity doesn't have any sample data or its model is removed from the model cache. - - -#### Request - -```json -GET _plugins/_anomaly_detection/detectors//_profile?_all=true -{ - "entity": [ - { - "name": "host", - "value": "i-00f28ec1eb8997686" - } - ] -} -``` - -#### Sample Responses - -```json -{ - "category_field": "host", - "value": "i-00f28ec1eb8997686", - "is_active": true, - "last_active_timestamp": 1604026394879, - "last_sample_timestamp": 1604026394879, - "init_progress": { - "percentage": "100%" - }, - "model": { - "model_id": "TFUdd3UBBwIAGQeRh5IS_entity_i-00f28ec1eb8997686", - "model_size_in_bytes": 712480, - "node_id": "MQ-bTBW3Q2uU_2zX3pyEQg" - }, - "state": "RUNNING" -} -``` - -To get profile information for only historical analysis, specify `ad_task`. -Specifying `_all` is an expensive operation for multi-category high cardinality detectors. - -#### Request - -```json -GET _plugins/_anomaly_detection/detectors//_profile?_all -GET _plugins/_anomaly_detection/detectors//_profile/ad_task -``` - -#### Sample Responses - -```json -{ - "ad_task": { - "ad_task": { - "task_id": "JXxyG3YBv5IHYYfMlFS2", - "last_update_time": 1606778263543, - "state": "STOPPED", - "detector_id": "SwvxCHYBPhugfWD9QAL6", - "task_progress": 0.010480972, - "init_progress": 1, - "current_piece": 1578140400000, - "execution_start_time": 1606778262709, - "is_latest": true, - "task_type": "HISTORICAL", - "detector": { - "name": "historical_test1", - "description": "test", - "time_field": "timestamp", - "indices": [ - "server_log" - ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 5, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "zgvyCHYBPhugfWD9Ap_F", - "feature_name": "sum", - "feature_enabled": true, - "aggregation_query": { - "sum": { - "sum": { - "field": "value" - } - } - } - }, - { - "feature_id": "zwvyCHYBPhugfWD9Ap_G", - "feature_name": "max", - "feature_enabled": true, - "aggregation_query": { - "max": { - "max": { - "field": "value" - } - } - } - } - ], - "ui_metadata": { - "features": { - "max": { - "aggregationBy": "max", - "aggregationOf": "value", - "featureType": "simple_aggs" - }, - "sum": { - "aggregationBy": "sum", - "aggregationOf": "value", - "featureType": "simple_aggs" - } - }, - "filters": [], - "filterType": "simple_filter" - }, - "last_update_time": 1606467935713, - "detector_type": "HISTORICAL_SIGLE_ENTITY", - "detection_date_range": { - "start_time": 1577840400000, - "end_time": 1606463775000 - } - } - }, - "shingle_size": 8, - "rcf_total_updates": 1994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "node_id": "Q9yznwxvTz-yJxtz7rJlLg" - } -} -``` - ---- diff --git a/_monitoring-plugins/ad/index.md b/_monitoring-plugins/ad/index.md index d41f6455..6abfe5ac 100644 --- a/_monitoring-plugins/ad/index.md +++ b/_monitoring-plugins/ad/index.md @@ -79,7 +79,7 @@ To get the entity model size of a detector, use the [profile detector API]({{sit This formula provides a good starting point, but make sure to test with a representative workload. {: .note } -For example, for a cluster with 3 data nodes, each with 8G of JVM heap size, a maximum memory percentage of 10% (default), and the entity model size of the detector as 1MB: the total number of unique entities supported is (8.096 * 10^9 * 0.1 / 1M ) * 3 = 2429. +For example, for a cluster with three data nodes, each with 8 GB of JVM heap size, a maximum memory percentage of 10% (default), and the entity model size of the detector as 1MB: the total number of unique entities supported is (8.096 * 10^9 * 0.1 / 1 MB ) * 3 = 2429. If you set the total number of unique entities higher than this number that you calculate (in this case: 2429), the anomaly detector makes its best effort to model the extra entities. The detector prioritizes entities that occur more often and are more recent. diff --git a/_monitoring-plugins/ad/settings.md b/_monitoring-plugins/ad/settings.md index 5aa73dc4..e63ab922 100644 --- a/_monitoring-plugins/ad/settings.md +++ b/_monitoring-plugins/ad/settings.md @@ -40,7 +40,7 @@ Setting | Default | Description `plugins.anomaly_detection.batch_task_piece_size` | 1,000 | The date range for a historical task is split into smaller pieces and the anomaly detection plugin runs the task piece by piece. Each piece contains 1,000 detection intervals by default. For example, if detector interval is 1 minute and one piece is 1,000 minutes, the feature data is queried every 1,000 minutes. You can change this setting from 1 to 10,000. `plugins.anomaly_detection.batch_task_piece_interval_seconds` | 5 | Add a time interval between two pieces of the same historical analysis task. This interval prevents the task from consuming too much of the available resources and starving other operations like search and bulk index. You can change this setting from 1 to 600 seconds. `plugins.anomaly_detection.max_top_entities_for_historical_analysis` | 1,000 | The maximum number of top entities that you run for a high cardinality detector historical analysis. The range is from 1 to 10,000. -`plugins.anomaly_detection.max_running_entities_per_detector_for_historical_analysis` | 10 | The number of entity tasks that you can run in parallel for a single high cardinality detector. The task slots available on your cluster also impact how many entities run in parallel. If a cluster has 3 data nodes, each data node has 10 task slots by default. Say you already have two high cardinality detectors and each of them run 10 entities. If you start a single-flow detector that takes 1 task slot, the number of task slots available is 10 * 3 - 10 * 2 - 1 = 9. if you now start a new high cardinality detector, the detector can only run 9 entities in parallel and not 10. You can tune this value from 1 to 1,000 based on your cluster's capability. If you set a higher value, the anomaly detection plugin runs historical analysis faster but also consumes more resources. +`plugins.anomaly_detection.max_running_entities_per_detector_for_historical_analysis` | 10 | The number of entity tasks that you can run in parallel for a single high cardinality detector. The task slots available on your cluster also impact how many entities run in parallel. If a cluster has 3 data nodes, each data node has 10 task slots by default. Say you already have two high cardinality detectors and each of them run 10 entities. If you start a single-flow detector that takes 1 task slot, the number of task slots available is 10 * 3 - 10 * 2 - 1 = 9. If you now start a new high cardinality detector, the detector can only run 9 entities in parallel and not 10. You can tune this value from 1 to 1,000 based on your cluster's capability. If you set a higher value, the anomaly detection plugin runs historical analysis faster but also consumes more resources. `plugins.anomaly_detection.max_cached_deleted_tasks` | 1,000 | You can rerun historical analysis for a single detector as many times as you like. The anomaly detection plugin only keeps a limited number of old tasks, by default 1 old task. If you run historical analysis three times for a detector, the oldest task is deleted. Because historical analysis generates a number of anomaly results in a short span of time, it's necessary to clean up anomaly results for a deleted task. With this field, you can configure how many deleted tasks you can cache at most. The plugin cleans up a task's results when it's deleted. If the plugin fails to do this cleanup, it adds the task's results into a cache and an hourly cron job performs the cleanup. You can use this setting to limit how many old tasks are put into cache to avoid a DDoS attack. After an hour, if still you find an old task result in the cache, use the [delete detector results API]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/api/#delete-detector-results) to delete the task result manually. You can tune this setting from 1 to 10,000. `plugins.anomaly_detection.delete_anomaly_result_when_delete_detector` | False | Whether the anomaly detection plugin deletes the anomaly result when you delete a detector. If you want to save some disk space, especially if you've high cardinality detectors generating a lot of results, set this field to true. Alternatively, you can use the [delete detector results API]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/api/#delete-detector-results) to manually delete the results. `plugins.anomaly_detection.dedicated_cache_size` | 10 | If the real-time analysis of a high cardinality detector starts successfully, the anomaly detection plugin guarantees keeping 10 (dynamically adjustable via this setting) entities' models in memory per node. If the number of entities exceeds this limit, the plugin puts the extra entities' models in a memory space shared by all detectors. The actual number of entities varies based on the memory that you've available and the frequencies of the entities. If you'd like the plugin to guarantee keeping more entities' models in memory and if you're cluster has sufficient memory, you can increase this setting value. From 79737566a993993cf5f29ebd90000635debe804e Mon Sep 17 00:00:00 2001 From: Liz Snyder Date: Tue, 5 Oct 2021 08:37:21 -0700 Subject: [PATCH 070/229] Add stats APIs --- _replication-plugin/api.md | 146 ++++++++++++++++++++++++++++- _replication-plugin/auto-follow.md | 34 ++++++- _replication-plugin/get-started.md | 4 +- 3 files changed, 179 insertions(+), 5 deletions(-) diff --git a/_replication-plugin/api.md b/_replication-plugin/api.md index 58457c84..df4a34b3 100644 --- a/_replication-plugin/api.md +++ b/_replication-plugin/api.md @@ -83,6 +83,8 @@ POST /_plugins/_replication//_pause {} ``` +You can't resume replication after it's been paused for more than 12 hours. You must [stop replication]({{site.url}}{{site.baseurl}}/replication-plugin/api/#stop-replication), delete the follower index, and restart replication of the leader. + #### Sample response ```json @@ -142,7 +144,149 @@ GET /_plugins/_replication//_status ``` To include shard replication details in the response, add the `&verbose=true` parameter. -The leader and follower checkpoint values begin as negative integers and reflect the number of shards you have (-1 for one shard, -5 for five shards, and so on). The values increment to positive integers with each change that you make. For example, when you make a change on the leader index, the `leader_checkpoint` becomes `0`. The `follower_checkpoint` is initially still `-1` until the follower index pulls the change from the leader, at which point it increments to `0`. If the values are the same, it means the indices are fully synced. +The leader and follower checkpoint values begin as negative integers and reflect the shard count (-1 for one shard, -5 for five shards, and so on). The values increment toward positive integers with each change that you make. For example, when you make a change on the leader index, the `leader_checkpoint` becomes `0`. The `follower_checkpoint` is initially still `-1` until the follower index pulls the change from the leader, at which point it increments to `0`. If the values are the same, it means the indices are fully synced. + +## Get leader cluster stats +Introduced 1.1 +{: .label .label-purple } + +Gets information about replicated leader indices on a specified cluster. + +#### Request + +```json +GET /_plugins/_replication/leader_stats +``` + +#### Sample response + +```json +{ + "num_replicated_indices": 2, + "operations_read": 15, + "translog_size_bytes": 1355, + "operations_read_lucene": 0, + "operations_read_translog": 15, + "total_read_time_lucene_millis": 0, + "total_read_time_translog_millis": 659, + "bytes_read": 1000, + "index_stats":{ + "leader-index-1":{ + "operations_read": 7, + "translog_size_bytes": 639, + "operations_read_lucene": 0, + "operations_read_translog": 7, + "total_read_time_lucene_millis": 0, + "total_read_time_translog_millis": 353, + "bytes_read":466 + }, + "leader-index-2":{ + "operations_read": 8, + "translog_size_bytes": 716, + "operations_read_lucene": 0, + "operations_read_translog": 8, + "total_read_time_lucene_millis": 0, + "total_read_time_translog_millis": 306, + "bytes_read": 534 + } + } +} +``` + +## Get follower cluster stats +Introduced 1.1 +{: .label .label-purple } + +Gets information about follower (syncing) indices on a specified cluster. + +#### Request + +```json +GET /_plugins/_replication/follower_stats +``` + +#### Sample response + +```json +{ + "num_syncing_indices": 2, + "num_bootstrapping_indices": 0, + "num_paused_indices": 0, + "num_failed_indices": 0, + "num_shard_tasks": 2, + "num_index_tasks": 2, + "operations_written": 3, + "operations_read": 3, + "failed_read_requests": 0, + "throttled_read_requests": 0, + "failed_write_requests": 0, + "throttled_write_requests": 0, + "follower_checkpoint": 1, + "leader_checkpoint": 1, + "total_write_time_millis": 2290, + "index_stats":{ + "follower-index-1":{ + "operations_written": 2, + "operations_read": 2, + "failed_read_requests": 0, + "throttled_read_requests": 0, + "failed_write_requests": 0, + "throttled_write_requests": 0, + "follower_checkpoint": 1, + "leader_checkpoint": 1, + "total_write_time_millis": 1355 + }, + "follower-index-2":{ + "operations_written": 1, + "operations_read": 1, + "failed_read_requests": 0, + "throttled_read_requests": 0, + "failed_write_requests": 0, + "throttled_write_requests": 0, + "follower_checkpoint": 0, + "leader_checkpoint": 0, + "total_write_time_millis": 935 + } + } +} +``` + +## Get auto-follow stats +Introduced 1.1 +{: .label .label-purple } + +Gets information about auto-follow activity and any replication rules configured on the specified cluster. + +#### Request + +```json +GET /_plugins/_replication/autofollow_stats +``` + +#### Sample response + +```json +{ + "num_success_start_replication": 2, + "num_failed_start_replication": 0, + "num_failed_leader_calls": 0, + "failed_indices":[ + + ], + "autofollow_stats":[ + { + "name":"my-replication-rule", + "pattern":"movies*", + "num_success_start_replication": 2, + "num_failed_start_replication": 0, + "num_failed_leader_calls": 0, + "failed_indices":[ + + ] + } + ] +} +``` ## Update settings Introduced 1.1 diff --git a/_replication-plugin/auto-follow.md b/_replication-plugin/auto-follow.md index fd2e69b4..0098d4e4 100644 --- a/_replication-plugin/auto-follow.md +++ b/_replication-plugin/auto-follow.md @@ -24,9 +24,6 @@ If the security plugin is enabled, non-admin users need to be mapped to the appr Replication rules are a collection of patterns that you create against a single remote cluster. When you create a replication rule, it automatically starts replicating any *new* indices that match the pattern, but does not replicate matching indices that were previously created. -Make sure to note the names of all rules when you create them. The replication plugin currently does not include an API operation to retrieve a list of existing rules. -{: .tip } - Create a replication rule on the follower cluster: ```bash @@ -57,11 +54,42 @@ And confirm its replica shows up on the follower cluster: curl -XGET -u 'admin:admin' -k 'https://localhost:9200/_cat/indices?v' ``` +It might take several seconds for the index to appear. + ```bash health status index uuid pri rep docs.count docs.deleted store.size pri.store.size yellow open movies-0001 kHOxYYHxRMeszLjTD9rvSQ 1 1 0 0 208b 208b ``` +## Retrieve replication rules + +To retrieve a list of existing replication rules configured on a cluster, send the following request: + +```bash +curl -XGET -u 'admin:admin' -k 'https://localhost:9200/_plugins/_replication/autofollow_stats' + +{ + "num_success_start_replication": 1, + "num_failed_start_replication": 0, + "num_failed_leader_calls": 0, + "failed_indices":[ + + ], + "autofollow_stats":[ + { + "name":"my-replication-rule", + "pattern":"movies*", + "num_success_start_replication": 1, + "num_failed_start_replication": 0, + "num_failed_leader_calls": 0, + "failed_indices":[ + + ] + } + ] +} +``` + ## Delete a replication rule When you delete a replication rule, OpenSearch stops replicating *new* indices that match the pattern, but existing indices that the rule previously created continue to replicate. If you need to stop existing replication activity, use the [stop replication API operation]({{site.url}}{{site.baseurl}}/replication-plugin/api/#stop-replication). diff --git a/_replication-plugin/get-started.md b/_replication-plugin/get-started.md index 2ab9d755..5b0daace 100644 --- a/_replication-plugin/get-started.md +++ b/_replication-plugin/get-started.md @@ -189,7 +189,7 @@ curl -XGET -k -u 'admin:admin' 'https://localhost:9200/_plugins/_replication/fol Possible statuses are `SYNCING`, `BOOTSTRAPING`, `PAUSED`, and `REPLICATION NOT IN PROGRESS`. -The leader and follower checkpoint values begin as negative numbers and reflect the number of shards you have (-1 for one shard, -5 for five shards, and so on). The values increment with each change and illustrate how many updates the follower is behind the leader. If the indices are fully synced, the values are the same. +The leader and follower checkpoint values begin as negative numbers and reflect the shard count (-1 for one shard, -5 for five shards, and so on). The values increment with each change and illustrate how many updates the follower is behind the leader. If the indices are fully synced, the values are the same. To confirm that replication is actually happening, add a document to the leader index: @@ -246,6 +246,8 @@ curl -XPOST -k -H 'Content-Type: application/json' -u 'admin:admin' 'https://loc When replication resumes, the follower index picks up any changes that were made to the leader index while replication was paused. +Note that you can't resume replication after it's been paused for more than 12 hours. You must [stop replication]({{site.url}}{{site.baseurl}}/replication-plugin/api/#stop-replication), delete the follower index, and restart replication of the leader. + ## Stop replication Terminate replication of a specified index from the follower cluster: From c91b99037b5876b0c7bbddeafc9c4e7f0ec9d211 Mon Sep 17 00:00:00 2001 From: aetter Date: Tue, 5 Oct 2021 09:49:15 -0700 Subject: [PATCH 071/229] Add release highlights. --- version-history.md | 1 + 1 file changed, 1 insertion(+) diff --git a/version-history.md b/version-history.md index bb67d8b9..f5d8f3de 100644 --- a/version-history.md +++ b/version-history.md @@ -9,6 +9,7 @@ permalink: /version-history/ OpenSearch version | Release highlights | Release date :--- | :--- | :--- | :--- +[1.1.0](https://github.com/opensearch-project/opensearch-build/tree/main/release-notes/opensearch-release-notes-1.1.0.md) | Adds cross-cluster replication, security for Index Management, ARM support, bucket-level alerting, a CLI to help with upgrading from Elasticsearch OSS to OpenSearch, and enhancements to high cardinality data in the anomaly detection plugin. | 5 October 2021 [1.0.1](https://github.com/opensearch-project/opensearch-build/tree/main/release-notes/opensearch-release-notes-1.0.1.md) | Bug fixes. | 1 September 2021 [1.0.0](https://github.com/opensearch-project/opensearch-build/tree/main/release-notes/opensearch-release-notes-1.0.0.md) | General availability release. Adds compatibility setting for clients that require a version check before connecting. | 12 July 2021 [1.0.0-rc1](https://github.com/opensearch-project/opensearch-build/tree/main/release-notes/opensearch-release-notes-1.0.0-rc1.md) | First release candidate. | 7 June 2021 From 31003b1523e77e0972a453d49cfdb0559021b475 Mon Sep 17 00:00:00 2001 From: Miki Date: Tue, 17 Aug 2021 14:19:08 -0700 Subject: [PATCH 072/229] Add versioned documentation Reduce font size and expand width Signed-off-by: Miki --- _data/versions.json | 3 + _includes/head_custom.html | 6 + _layouts/default.html | 4 + _sass/color_schemes/opensearch.scss | 2 + _sass/custom/custom.scss | 20 +++- assets/js/_version-selector.js | 176 ++++++++++++++++++++++++++++ assets/js/version-selector.tpl | 9 ++ 7 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 _data/versions.json create mode 100644 assets/js/_version-selector.js create mode 100644 assets/js/version-selector.tpl diff --git a/_data/versions.json b/_data/versions.json new file mode 100644 index 00000000..9e5f8d81 --- /dev/null +++ b/_data/versions.json @@ -0,0 +1,3 @@ +{ + "current": "1.0" +} \ No newline at end of file diff --git a/_includes/head_custom.html b/_includes/head_custom.html index 91ee8a17..1a18be03 100755 --- a/_includes/head_custom.html +++ b/_includes/head_custom.html @@ -6,3 +6,9 @@ {% endif %} + +{% if jekyll.environment == "development" %} + +{% else %} + +{% endif %} diff --git a/_layouts/default.html b/_layouts/default.html index d433719d..c5408662 100755 --- a/_layouts/default.html +++ b/_layouts/default.html @@ -57,6 +57,10 @@ layout: table_wrappers