iSharkFly-Docs
/
opensearch-docs-cn
mirror of https://github.com/iSharkFly-Docs/opensearch-docs-cn
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
opensearch-docs-cn/_monitoring-plugins/ad/api.md

3180 lines
68 KiB
Markdown
Raw Normal View History

Breaks site
2021-05-28 13:48:19 -04:00
---
layout: default
title: Anomaly detection API
parent: Anomaly detection
nav_order: 1
---
# Anomaly detection API
Use these anomaly detection operations to programmatically create and manage detectors.
---
#### Table of contents
- TOC
{:toc}
---
## Create anomaly detector
Add "introduced" labels for APIs
2021-07-26 19:14:22 -04:00
Introduced 1.0
{: .label .label-purple }
Breaks site
2021-05-28 13:48:19 -04:00
Creates an anomaly detector.
incorporated feedback
2021-10-05 17:43:31 -04:00
This command creates a single-entity detector named `test-detector` that finds anomalies based on the sum of the `value` field:
Breaks site
2021-05-28 13:48:19 -04:00
#### Request
```json
POST _plugins/_anomaly_detection/detectors
{
"name": "test-detector",
"description": "Test detector",
"time_field": "timestamp",
"indices": [
updated requests and responses
2021-10-05 05:36:58 -04:00
"server_log*"
Breaks site
2021-05-28 13:48:19 -04:00
],
"feature_attributes": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_name": "test",
Breaks site
2021-05-28 13:48:19 -04:00
"feature_enabled": true,
"aggregation_query": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"test": {
Breaks site
2021-05-28 13:48:19 -04:00
"sum": {
"field": "value"
}
}
}
}
],
"filter_query": {
"bool": {
"filter": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"range": {
"value": {
"gt": 1
}
Breaks site
2021-05-28 13:48:19 -04:00
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"detection_interval": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
}
}
}
```
#### Sample response
```json
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"_id": "VEHKTXwBwf_U8gjUXY2s",
Breaks site
2021-05-28 13:48:19 -04:00
"_version": 1,
updated requests and responses
2021-10-05 05:36:58 -04:00
"_seq_no": 5,
Breaks site
2021-05-28 13:48:19 -04:00
"anomaly_detector": {
"name": "test-detector",
"description": "Test detector",
"time_field": "timestamp",
"indices": [
updated requests and responses
2021-10-05 05:36:58 -04:00
"server_log*"
Breaks site
2021-05-28 13:48:19 -04:00
],
"filter_query": {
"bool": {
"filter": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"range": {
"value": {
"from": 1,
"to": null,
"include_lower": false,
"include_upper": true,
"boost": 1
}
Breaks site
2021-05-28 13:48:19 -04:00
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"detection_interval": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"shingle_size": 8,
Breaks site
2021-05-28 13:48:19 -04:00
"schema_version": 0,
"feature_attributes": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_id": "U0HKTXwBwf_U8gjUXY2m",
"feature_name": "test",
Breaks site
2021-05-28 13:48:19 -04:00
"feature_enabled": true,
"aggregation_query": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"test": {
Breaks site
2021-05-28 13:48:19 -04:00
"sum": {
"field": "value"
}
}
}
}
updated requests and responses
2021-10-05 05:36:58 -04:00
],
"last_update_time": 1633392680364,
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
"own_index",
"all_access"
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
},
"detector_type": "SINGLE_ENTITY"
},
"_primary_term": 1
Breaks site
2021-05-28 13:48:19 -04:00
}
```
minor changes
2021-10-05 15:41:37 -04:00
To create a high cardinality detector by specifying a category field:
Breaks site
2021-05-28 13:48:19 -04:00
#### Request
```json
POST _plugins/_anomaly_detection/detectors
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"name": "test-hc-detector",
"description": "Test detector",
"time_field": "timestamp",
Breaks site
2021-05-28 13:48:19 -04:00
"indices": [
updated requests and responses
2021-10-05 05:36:58 -04:00
"server_log*"
Breaks site
2021-05-28 13:48:19 -04:00
],
"feature_attributes": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_name": "test",
Breaks site
2021-05-28 13:48:19 -04:00
"feature_enabled": true,
"aggregation_query": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"test": {
"sum": {
"field": "value"
Breaks site
2021-05-28 13:48:19 -04:00
}
}
}
}
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"filter_query": {
"bool": {
"filter": [
{
"range": {
"value": {
"gt": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
Breaks site
2021-05-28 13:48:19 -04:00
}
},
"detection_interval": {
"period": {
"interval": 1,
updated requests and responses
2021-10-05 05:36:58 -04:00
"unit": "Minutes"
Breaks site
2021-05-28 13:48:19 -04:00
}
updated requests and responses
2021-10-05 05:36:58 -04:00
},
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"category_field": [
"ip"
]
Breaks site
2021-05-28 13:48:19 -04:00
}
```
#### Sample response
```json
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"_id": "b0HRTXwBwf_U8gjUw43R",
Breaks site
2021-05-28 13:48:19 -04:00
"_version": 1,
updated requests and responses
2021-10-05 05:36:58 -04:00
"_seq_no": 6,
Breaks site
2021-05-28 13:48:19 -04:00
"anomaly_detector": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"name": "test-hc-detector",
"description": "Test detector",
"time_field": "timestamp",
Breaks site
2021-05-28 13:48:19 -04:00
"indices": [
updated requests and responses
2021-10-05 05:36:58 -04:00
"server_log*"
Breaks site
2021-05-28 13:48:19 -04:00
],
"filter_query": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"bool": {
"filter": [
{
"range": {
"value": {
"from": 1,
"to": null,
"include_lower": false,
"include_upper": true,
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
Breaks site
2021-05-28 13:48:19 -04:00
"boost": 1
}
},
"detection_interval": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"window_delay": {
"period": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"interval": 1,
"unit": "Minutes"
Breaks site
2021-05-28 13:48:19 -04:00
}
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"shingle_size": 8,
"schema_version": 0,
Breaks site
2021-05-28 13:48:19 -04:00
"feature_attributes": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_id": "bkHRTXwBwf_U8gjUw43K",
"feature_name": "test",
Breaks site
2021-05-28 13:48:19 -04:00
"feature_enabled": true,
"aggregation_query": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"test": {
"sum": {
"field": "value"
Breaks site
2021-05-28 13:48:19 -04:00
}
}
}
}
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"last_update_time": 1633393165265,
Breaks site
2021-05-28 13:48:19 -04:00
"category_field": [
updated requests and responses
2021-10-05 05:36:58 -04:00
"ip"
],
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
"own_index",
"all_access"
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
},
"detector_type": "MULTI_ENTITY"
Breaks site
2021-05-28 13:48:19 -04:00
},
"_primary_term": 1
}
```
updated requests and responses
2021-10-05 05:36:58 -04:00
You can specify a maximum of two category fields:
```json
"category_field": [
"ip"
]
```
```json
"category_field": [
"ip", "error_type"
]
```
Breaks site
2021-05-28 13:48:19 -04:00
You can specify the following options.
Options | Description | Type | Required
:--- | :--- |:--- |:--- |
`name` | The name of the detector. | `string` | Yes
`description` | A description of the detector. | `string` | Yes
`time_field` | The name of the time field. | `string` | Yes
`indices` | A list of indices to use as the data source. | `list` | Yes
`feature_attributes` | Specify a `feature_name`, set the `enabled` parameter to `true`, and specify an aggregation query. | `list` | Yes
`filter_query` | Provide an optional filter query for your feature. | `object` | No
`detection_interval` | The time interval for your anomaly detector. | `object` | Yes
`window_delay` | Add extra processing time for data collection. | `object` | No
`category_field` | Categorizes or slices data with a dimension. Similar to `GROUP BY` in SQL. | `list` | No
---
incorporated feedback
2021-10-04 17:11:45 -04:00
## Get detector
Add "introduced" labels for APIs
2021-07-26 19:14:22 -04:00
Introduced 1.0
{: .label .label-purple }
Breaks site
2021-05-28 13:48:19 -04:00
incorporated feedback
2021-10-04 17:11:45 -04:00
Returns all information about a detector based on the `detector_id`.
Breaks site
2021-05-28 13:48:19 -04:00
#### Request
```json
incorporated feedback
2021-10-04 17:11:45 -04:00
GET _plugins/_anomaly_detection/detectors/<detectorId>
```
#### Sample response
```json
Breaks site
2021-05-28 13:48:19 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"_id": "VEHKTXwBwf_U8gjUXY2s",
"_version": 1,
"_primary_term": 1,
"_seq_no": 5,
"anomaly_detector": {
"name": "test-detector",
"description": "Test detector",
"time_field": "timestamp",
"indices": [
"server_log*"
incorporated feedback
2021-10-04 17:11:45 -04:00
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"filter_query": {
"bool": {
"filter": [
incorporated feedback
2021-10-04 17:11:45 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"range": {
"value": {
"from": 1,
"to": null,
"include_lower": false,
"include_upper": true,
"boost": 1
}
incorporated feedback
2021-10-04 17:11:45 -04:00
}
}
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"adjust_pure_negative": true,
"boost": 1
incorporated feedback
2021-10-04 17:11:45 -04:00
}
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"detection_interval": {
"period": {
"interval": 1,
"unit": "Minutes"
incorporated feedback
2021-10-04 17:11:45 -04:00
}
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
incorporated feedback
2021-10-04 17:11:45 -04:00
}
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"shingle_size": 8,
"schema_version": 0,
"feature_attributes": [
incorporated feedback
2021-10-04 17:11:45 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_id": "U0HKTXwBwf_U8gjUXY2m",
"feature_name": "test",
"feature_enabled": true,
"aggregation_query": {
"test": {
"sum": {
"field": "value"
incorporated feedback
2021-10-04 17:11:45 -04:00
}
}
}
}
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"last_update_time": 1633392680364,
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
"own_index",
"all_access"
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
},
"detector_type": "SINGLE_ENTITY"
incorporated feedback
2021-10-04 17:11:45 -04:00
}
}
```
minor changes
2021-10-05 15:41:37 -04:00
Use `job=true` to get real-time analysis task information.
incorporated feedback
2021-10-04 17:11:45 -04:00
#### Request
```json
GET _plugins/_anomaly_detection/detectors/<detectorId>?job=true
```
#### Sample response
```json
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"_id": "VEHKTXwBwf_U8gjUXY2s",
incorporated feedback
2021-10-04 17:11:45 -04:00
"_version": 1,
"_primary_term": 1,
updated requests and responses
2021-10-05 05:36:58 -04:00
"_seq_no": 5,
incorporated feedback
2021-10-04 17:11:45 -04:00
"anomaly_detector": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"name": "test-detector",
"description": "Test detector",
ad 1.1
2021-10-01 14:22:47 -04:00
"time_field": "timestamp",
"indices": [
updated requests and responses
2021-10-05 05:36:58 -04:00
"server_log*"
ad 1.1
2021-10-01 14:22:47 -04:00
],
incorporated feedback
2021-10-04 17:11:45 -04:00
"filter_query": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"bool": {
"filter": [
{
"range": {
"value": {
"from": 1,
"to": null,
"include_lower": false,
"include_upper": true,
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
incorporated feedback
2021-10-04 17:11:45 -04:00
"boost": 1
}
},
ad 1.1
2021-10-01 14:22:47 -04:00
"detection_interval": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
incorporated feedback
2021-10-04 17:11:45 -04:00
"shingle_size": 8,
"schema_version": 0,
ad 1.1
2021-10-01 14:22:47 -04:00
"feature_attributes": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_id": "U0HKTXwBwf_U8gjUXY2m",
"feature_name": "test",
"feature_enabled": true,
ad 1.1
2021-10-01 14:22:47 -04:00
"aggregation_query": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"test": {
ad 1.1
2021-10-01 14:22:47 -04:00
"sum": {
"field": "value"
}
}
}
}
incorporated feedback
2021-10-04 17:11:45 -04:00
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"last_update_time": 1633392680364,
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
"own_index",
"all_access"
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
},
"detector_type": "SINGLE_ENTITY"
incorporated feedback
2021-10-04 17:11:45 -04:00
},
"anomaly_detector_job": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"name": "VEHKTXwBwf_U8gjUXY2s",
incorporated feedback
2021-10-04 17:11:45 -04:00
"schedule": {
"interval": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"start_time": 1633393656357,
incorporated feedback
2021-10-04 17:11:45 -04:00
"period": 1,
"unit": "Minutes"
}
},
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"enabled": true,
"enabled_time": 1633393656357,
"last_update_time": 1633393656357,
incorporated feedback
2021-10-04 17:11:45 -04:00
"lock_duration_seconds": 60,
updated requests and responses
2021-10-05 05:36:58 -04:00
"user": {
"name": "admin",
"backend_roles": [
"admin"
incorporated feedback
2021-10-04 17:11:45 -04:00
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"roles": [
"own_index",
"all_access"
incorporated feedback
2021-10-04 17:11:45 -04:00
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
incorporated feedback
2021-10-04 17:11:45 -04:00
}
ad 1.1
2021-10-01 14:22:47 -04:00
}
Breaks site
2021-05-28 13:48:19 -04:00
}
```
incorporated feedback
2021-10-05 17:43:31 -04:00
Use `task=true` to get information for both real-time and historical analysis task information.
incorporated feedback
2021-10-04 17:11:45 -04:00
#### Request
```json
GET _plugins/_anomaly_detection/detectors/<detectorId>?task=true
```
Breaks site
2021-05-28 13:48:19 -04:00
#### Sample response
```json
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"_id": "VEHKTXwBwf_U8gjUXY2s",
incorporated feedback
2021-10-04 17:11:45 -04:00
"_version": 1,
updated requests and responses
2021-10-05 05:36:58 -04:00
"_primary_term": 1,
"_seq_no": 5,
Breaks site
2021-05-28 13:48:19 -04:00
"anomaly_detector": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"name": "test-detector",
"description": "Test detector",
Breaks site
2021-05-28 13:48:19 -04:00
"time_field": "timestamp",
"indices": [
updated requests and responses
2021-10-05 05:36:58 -04:00
"server_log*"
Breaks site
2021-05-28 13:48:19 -04:00
],
"filter_query": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"bool": {
"filter": [
{
"range": {
"value": {
"from": 1,
"to": null,
"include_lower": false,
"include_upper": true,
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
Breaks site
2021-05-28 13:48:19 -04:00
"boost": 1
}
},
"detection_interval": {
"period": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"interval": 1,
incorporated feedback
2021-10-04 17:11:45 -04:00
"unit": "Minutes"
Breaks site
2021-05-28 13:48:19 -04:00
}
},
"window_delay": {
"period": {
"interval": 1,
incorporated feedback
2021-10-04 17:11:45 -04:00
"unit": "Minutes"
Breaks site
2021-05-28 13:48:19 -04:00
}
},
incorporated feedback
2021-10-04 17:11:45 -04:00
"shingle_size": 8,
Breaks site
2021-05-28 13:48:19 -04:00
"schema_version": 0,
"feature_attributes": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_id": "U0HKTXwBwf_U8gjUXY2m",
"feature_name": "test",
Breaks site
2021-05-28 13:48:19 -04:00
"feature_enabled": true,
"aggregation_query": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"test": {
Breaks site
2021-05-28 13:48:19 -04:00
"sum": {
"field": "value"
}
}
}
}
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"last_update_time": 1633392680364,
incorporated feedback
2021-10-04 17:11:45 -04:00
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
updated requests and responses
2021-10-05 05:36:58 -04:00
"own_index",
"all_access"
incorporated feedback
2021-10-04 17:11:45 -04:00
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"detector_type": "SINGLE_ENTITY"
incorporated feedback
2021-10-04 17:11:45 -04:00
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"realtime_detection_task": {
"task_id": "nkTZTXwBjd8s6RK4QlMq",
"last_update_time": 1633393776375,
incorporated feedback
2021-10-04 17:11:45 -04:00
"started_by": "admin",
updated requests and responses
2021-10-05 05:36:58 -04:00
"error": "",
"state": "RUNNING",
"detector_id": "VEHKTXwBwf_U8gjUXY2s",
"task_progress": 0,
incorporated feedback
2021-10-04 17:11:45 -04:00
"init_progress": 1,
updated requests and responses
2021-10-05 05:36:58 -04:00
"execution_start_time": 1633393656362,
incorporated feedback
2021-10-04 17:11:45 -04:00
"is_latest": true,
updated requests and responses
2021-10-05 05:36:58 -04:00
"task_type": "REALTIME_SINGLE_ENTITY",
"coordinating_node": "SWD7ihu9TaaW1zKwFZNVNg",
incorporated feedback
2021-10-04 17:11:45 -04:00
"detector": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"name": "test-detector",
"description": "Test detector",
incorporated feedback
2021-10-04 17:11:45 -04:00
"time_field": "timestamp",
"indices": [
updated requests and responses
2021-10-05 05:36:58 -04:00
"server_log*"
incorporated feedback
2021-10-04 17:11:45 -04:00
],
"filter_query": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"bool": {
"filter": [
{
"range": {
"value": {
"from": 1,
"to": null,
"include_lower": false,
"include_upper": true,
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
incorporated feedback
2021-10-04 17:11:45 -04:00
"boost": 1
}
},
"detection_interval": {
"period": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"interval": 1,
incorporated feedback
2021-10-04 17:11:45 -04:00
"unit": "Minutes"
}
},
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"shingle_size": 8,
"schema_version": 0,
"feature_attributes": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_id": "U0HKTXwBwf_U8gjUXY2m",
"feature_name": "test",
incorporated feedback
2021-10-04 17:11:45 -04:00
"feature_enabled": true,
"aggregation_query": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"test": {
incorporated feedback
2021-10-04 17:11:45 -04:00
"sum": {
"field": "value"
}
}
}
}
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"last_update_time": 1633392680364,
incorporated feedback
2021-10-04 17:11:45 -04:00
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
updated requests and responses
2021-10-05 05:36:58 -04:00
"own_index",
"all_access"
incorporated feedback
2021-10-04 17:11:45 -04:00
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"detector_type": "SINGLE_ENTITY"
incorporated feedback
2021-10-04 17:11:45 -04:00
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"estimated_minutes_left": 0,
incorporated feedback
2021-10-04 17:11:45 -04:00
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
updated requests and responses
2021-10-05 05:36:58 -04:00
"own_index",
"all_access"
incorporated feedback
2021-10-04 17:11:45 -04:00
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
Breaks site
2021-05-28 13:48:19 -04:00
}
updated requests and responses
2021-10-05 05:36:58 -04:00
},
"historical_analysis_task": {
"task_id": "99DaTXwB6HknB84StRN1",
"last_update_time": 1633393797040,
"started_by": "admin",
"state": "RUNNING",
"detector_id": "VEHKTXwBwf_U8gjUXY2s",
"task_progress": 0.89285713,
"init_progress": 1,
"current_piece": 1633328940000,
"execution_start_time": 1633393751412,
"is_latest": true,
"task_type": "HISTORICAL_SINGLE_ENTITY",
"coordinating_node": "SWD7ihu9TaaW1zKwFZNVNg",
"worker_node": "2Z4q22BySEyzakYt_A0A2A",
"detector": {
"name": "test-detector",
"description": "Test detector",
"time_field": "timestamp",
"indices": [
"server_log*"
],
"filter_query": {
"bool": {
"filter": [
{
"range": {
"value": {
"from": 1,
"to": null,
"include_lower": false,
"include_upper": true,
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"detection_interval": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"shingle_size": 8,
"schema_version": 0,
"feature_attributes": [
{
"feature_id": "U0HKTXwBwf_U8gjUXY2m",
"feature_name": "test",
"feature_enabled": true,
"aggregation_query": {
"test": {
"sum": {
"field": "value"
}
}
}
}
],
"last_update_time": 1633392680364,
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
"own_index",
"all_access"
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
},
"detector_type": "SINGLE_ENTITY"
},
"detection_date_range": {
"start_time": 1632788951329,
"end_time": 1633393751329
},
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
"own_index",
"all_access"
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
}
}
}
```
---
## Update detector
Introduced 1.0
{: .label .label-purple }
Updates a detector with any changes, including the description or adding or removing of features.
To update a detector, you need to first stop both real-time detection and historical analysis.
You can't update a category field.
{: .note }
#### Request
```json
PUT _plugins/_anomaly_detection/detectors/<detectorId>
{
"name": "test-detector",
"description": "Test update detector",
"time_field": "timestamp",
"indices": [
"server_log*"
],
"feature_attributes": [
incorporated feedback
2021-10-04 17:11:45 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_name": "test",
incorporated feedback
2021-10-04 17:11:45 -04:00
"feature_enabled": true,
"aggregation_query": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"test": {
incorporated feedback
2021-10-04 17:11:45 -04:00
"sum": {
"field": "value"
}
}
}
}
],
"filter_query": {
"bool": {
"filter": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"range": {
"value": {
"gt": 1
}
incorporated feedback
2021-10-04 17:11:45 -04:00
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"detection_interval": {
"period": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"interval": 1,
"unit": "Minutes"
incorporated feedback
2021-10-04 17:11:45 -04:00
}
},
"window_delay": {
"period": {
"interval": 1,
updated requests and responses
2021-10-05 05:36:58 -04:00
"unit": "Minutes"
incorporated feedback
2021-10-04 17:11:45 -04:00
}
}
ad 1.1
2021-10-01 14:22:47 -04:00
}
```
Breaks site
2021-05-28 13:48:19 -04:00
#### Sample response
```json
incorporated feedback
2021-10-04 17:11:45 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"_id": "VEHKTXwBwf_U8gjUXY2s",
"_version": 2,
"_seq_no": 7,
"anomaly_detector": {
"name": "test-detector",
"description": "Test update detector",
"time_field": "timestamp",
"indices": [
"server_log*"
incorporated feedback
2021-10-04 17:11:45 -04:00
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"filter_query": {
"bool": {
"filter": [
incorporated feedback
2021-10-04 17:11:45 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"range": {
"value": {
"from": 1,
"to": null,
"include_lower": false,
"include_upper": true,
"boost": 1
}
incorporated feedback
2021-10-04 17:11:45 -04:00
}
}
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"adjust_pure_negative": true,
"boost": 1
incorporated feedback
2021-10-04 17:11:45 -04:00
}
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"detection_interval": {
"period": {
"interval": 1,
"unit": "Minutes"
incorporated feedback
2021-10-04 17:11:45 -04:00
}
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
incorporated feedback
2021-10-04 17:11:45 -04:00
}
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"shingle_size": 8,
"schema_version": 0,
"feature_attributes": [
incorporated feedback
2021-10-04 17:11:45 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_id": "3kHiTXwBwf_U8gjUlY15",
"feature_name": "test",
"feature_enabled": true,
"aggregation_query": {
"test": {
"sum": {
"field": "value"
incorporated feedback
2021-10-04 17:11:45 -04:00
}
}
}
}
updated requests and responses
2021-10-05 05:36:58 -04:00
],
"last_update_time": 1633394267522,
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
"own_index",
"all_access"
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
},
"detector_type": "SINGLE_ENTITY"
},
"_primary_term": 1
}
```
---
## Delete detector
Introduced 1.0
{: .label .label-purple }
Deletes a detector based on the `detector_id`.
To delete a detector, you need to first stop both real-time detection and historical analysis.
#### Request
```json
DELETE _plugins/_anomaly_detection/detectors/<detectorId>
```
#### Sample response
```json
{
"_index": ".opendistro-anomaly-detectors",
"_type": "_doc",
"_id": "70TxTXwBjd8s6RK4j1Pj",
"_version": 2,
"result": "deleted",
"forced_refresh": true,
"_shards": {
"total": 2,
"successful": 2,
"failed": 0
},
"_seq_no": 9,
"_primary_term": 1
incorporated feedback
2021-10-04 17:11:45 -04:00
}
ad 1.1
2021-10-01 14:22:47 -04:00
```
Breaks site
2021-05-28 13:48:19 -04:00
---
incorporated feedback
2021-10-04 17:11:45 -04:00
## Preview detector
Add "introduced" labels for APIs
2021-07-26 19:14:22 -04:00
Introduced 1.0
{: .label .label-purple }
Breaks site
2021-05-28 13:48:19 -04:00
incorporated feedback
2021-10-04 17:11:45 -04:00
Passes a date range to the anomaly detector to return any anomalies within that date range.
Breaks site
2021-05-28 13:48:19 -04:00
incorporated feedback
2021-10-05 17:43:31 -04:00
To preview a single-entity detector:
updated requests and responses
2021-10-05 05:36:58 -04:00
Breaks site
2021-05-28 13:48:19 -04:00
#### Request
```json
incorporated feedback
2021-10-04 17:11:45 -04:00
POST _plugins/_anomaly_detection/detectors/<detectorId>/_preview
Breaks site
2021-05-28 13:48:19 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"period_start": 1633048868000,
"period_end": 1633394468000,
incorporated feedback
2021-10-04 17:11:45 -04:00
"detector": {
"name": "test-detector",
updated requests and responses
2021-10-05 05:36:58 -04:00
"description": "Test update detector",
incorporated feedback
2021-10-04 17:11:45 -04:00
"time_field": "timestamp",
"indices": [
updated requests and responses
2021-10-05 05:36:58 -04:00
"server_log*"
],
"feature_attributes": [
{
"feature_name": "test",
"feature_enabled": true,
"aggregation_query": {
"test": {
"sum": {
"field": "value"
}
}
}
}
incorporated feedback
2021-10-04 17:11:45 -04:00
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"filter_query": {
"bool": {
"filter": [
{
"range": {
"value": {
"gt": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
incorporated feedback
2021-10-04 17:11:45 -04:00
"detection_interval": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
}
updated requests and responses
2021-10-05 05:36:58 -04:00
}
Breaks site
2021-05-28 13:48:19 -04:00
}
}
```
#### Sample response
```json
{
incorporated feedback
2021-10-04 17:11:45 -04:00
"anomaly_result": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"detector_id": null,
"data_start_time": 1633049280000,
"data_end_time": 1633049340000,
"schema_version": 0,
incorporated feedback
2021-10-04 17:11:45 -04:00
"feature_data": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_id": "8EHmTXwBwf_U8gjU0Y0u",
"feature_name": "test",
"data": 0
Breaks site
2021-05-28 13:48:19 -04:00
}
incorporated feedback
2021-10-04 17:11:45 -04:00
],
"anomaly_grade": 0,
updated requests and responses
2021-10-05 05:36:58 -04:00
"confidence": 0
},
incorporated feedback
2021-10-04 17:11:45 -04:00
...
],
"anomaly_detector": {
"name": "test-detector",
updated requests and responses
2021-10-05 05:36:58 -04:00
"description": "Test update detector",
incorporated feedback
2021-10-04 17:11:45 -04:00
"time_field": "timestamp",
"indices": [
updated requests and responses
2021-10-05 05:36:58 -04:00
"server_log*"
incorporated feedback
2021-10-04 17:11:45 -04:00
],
"filter_query": {
"bool": {
"filter": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"range": {
"value": {
"from": 1,
"to": null,
"include_lower": false,
"include_upper": true,
"boost": 1
}
incorporated feedback
2021-10-04 17:11:45 -04:00
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"detection_interval": {
"period": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"interval": 1,
"unit": "Minutes"
incorporated feedback
2021-10-04 17:11:45 -04:00
}
},
"window_delay": {
"period": {
"interval": 1,
updated requests and responses
2021-10-05 05:36:58 -04:00
"unit": "Minutes"
incorporated feedback
2021-10-04 17:11:45 -04:00
}
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"shingle_size": 8,
incorporated feedback
2021-10-04 17:11:45 -04:00
"schema_version": 0,
"feature_attributes": [
Breaks site
2021-05-28 13:48:19 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_id": "8EHmTXwBwf_U8gjU0Y0u",
"feature_name": "test",
incorporated feedback
2021-10-04 17:11:45 -04:00
"feature_enabled": true,
"aggregation_query": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"test": {
incorporated feedback
2021-10-04 17:11:45 -04:00
"sum": {
"field": "value"
Breaks site
2021-05-28 13:48:19 -04:00
}
incorporated feedback
2021-10-04 17:11:45 -04:00
}
Breaks site
2021-05-28 13:48:19 -04:00
}
}
incorporated feedback
2021-10-04 17:11:45 -04:00
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"detector_type": "SINGLE_ENTITY"
Breaks site
2021-05-28 13:48:19 -04:00
}
}
```
incorporated feedback
2021-10-04 17:11:45 -04:00
If you specify a category field, each result is associated with an entity:
Breaks site
2021-05-28 13:48:19 -04:00
updated requests and responses
2021-10-05 05:36:58 -04:00
#### Request
incorporated feedback
2021-10-04 17:11:45 -04:00
```json
incorporated feedback
2021-10-05 17:43:31 -04:00
POST _plugins/_anomaly_detection/detectors/_preview
incorporated feedback
2021-10-04 17:11:45 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"period_start": 1633048868000,
"period_end": 1633394468000,
"detector": {
"name": "test-detector",
"description": "Test update detector",
"time_field": "timestamp",
"indices": [
"server_log*"
],
"feature_attributes": [
{
"feature_name": "test",
"feature_enabled": true,
"aggregation_query": {
"test": {
"sum": {
"field": "value"
}
}
}
}
],
"filter_query": {
"bool": {
"filter": [
{
"range": {
"value": {
"gt": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"detection_interval": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"category_field": [
"error_type"
]
}
}
```
#### Sample response
```json
{
"anomaly_result": [
{
"detector_id": null,
"data_start_time": 1633049280000,
"data_end_time": 1633049340000,
"schema_version": 0,
"feature_data": [
{
"feature_id": "tkTpTXwBjd8s6RK4DlOZ",
"feature_name": "test",
"data": 0
}
],
"anomaly_grade": 0,
"confidence": 0,
"entity": [
{
"name": "error_type",
"value": "error1"
}
]
},
...
],
"anomaly_detector": {
"name": "test-detector",
"description": "Test update detector",
"time_field": "timestamp",
"indices": [
"server_log*"
],
"filter_query": {
"bool": {
"filter": [
{
"range": {
"value": {
"from": 1,
"to": null,
"include_lower": false,
"include_upper": true,
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"detection_interval": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"shingle_size": 8,
"schema_version": 0,
"feature_attributes": [
{
"feature_id": "tkTpTXwBjd8s6RK4DlOZ",
"feature_name": "test",
"feature_enabled": true,
"aggregation_query": {
"test": {
"sum": {
"field": "value"
}
}
}
}
],
"category_field": [
"error_type"
],
"detector_type": "MULTI_ENTITY"
}
}
```
You can preview a detector with the detector ID:
```json
POST _plugins/_anomaly_detection/detectors/_preview
{
"detector_id": "VEHKTXwBwf_U8gjUXY2s",
"period_start": 1633048868000,
"period_end": 1633394468000
}
```
Or:
```json
POST _opendistro/_anomaly_detection/detectors/VEHKTXwBwf_U8gjUXY2s/_preview
{
"period_start": 1633048868000,
"period_end": 1633394468000
}
```
#### Sample response
```json
{
"anomaly_result": [
{
"detector_id": "VEHKTXwBwf_U8gjUXY2s",
"data_start_time": 1633049280000,
"data_end_time": 1633049340000,
"schema_version": 0,
"feature_data": [
{
"feature_id": "3kHiTXwBwf_U8gjUlY15",
"feature_name": "test",
"data": 0
}
],
"anomaly_grade": 0,
"confidence": 0,
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
"own_index",
"all_access"
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
}
},
...
],
"anomaly_detector": {
"name": "test-detector",
"description": "Test update detector",
"time_field": "timestamp",
"indices": [
"server_log*"
],
"filter_query": {
"bool": {
"filter": [
{
"range": {
"value": {
"from": 1,
"to": null,
"include_lower": false,
"include_upper": true,
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"detection_interval": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"shingle_size": 8,
"schema_version": 0,
"feature_attributes": [
{
"feature_id": "3kHiTXwBwf_U8gjUlY15",
"feature_name": "test",
"feature_enabled": true,
"aggregation_query": {
"test": {
"sum": {
"field": "value"
}
}
}
}
],
"last_update_time": 1633394267522,
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
"own_index",
"all_access"
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
},
"detector_type": "SINGLE_ENTITY"
}
}
incorporated feedback
2021-10-04 17:11:45 -04:00
```
---
## Start detector job
Introduced 1.0
{: .label .label-purple }
Starts a real-time or historical anomaly detector job.
updated requests and responses
2021-10-05 05:36:58 -04:00
To start a real-time detector job:
incorporated feedback
2021-10-04 17:11:45 -04:00
#### Request
```json
POST _plugins/_anomaly_detection/detectors/<detectorId>/_start
```
#### Sample response
```json
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"_id": "VEHKTXwBwf_U8gjUXY2s",
"_version": 3,
"_seq_no": 6,
"_primary_term": 1
incorporated feedback
2021-10-04 17:11:45 -04:00
}
```
updated requests and responses
2021-10-05 05:36:58 -04:00
The `_id` represents the real-time job ID, which is the same as the detector ID.
incorporated feedback
2021-10-04 17:11:45 -04:00
To start historical analysis:
```json
POST _plugins/_anomaly_detection/detectors/<detectorId>/_start
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"start_time": 1633048868000,
"end_time": 1633394468000
}
```
#### Sample response
```json
{
"_id": "f9DsTXwB6HknB84SoRTY",
"_version": 1,
"_seq_no": 958,
"_primary_term": 1
incorporated feedback
2021-10-04 17:11:45 -04:00
}
```
updated requests and responses
2021-10-05 05:36:58 -04:00
The `_id` represents the historical batch task ID, which is a random universally unique identifier (UUID).
incorporated feedback
2021-10-04 17:11:45 -04:00
---
## Stop detector job
Introduced 1.0
{: .label .label-purple }
Stops a real-time or historical anomaly detector job.
updated requests and responses
2021-10-05 05:36:58 -04:00
To stop a real-time detector job:
incorporated feedback
2021-10-04 17:11:45 -04:00
#### Request
```json
POST _plugins/_anomaly_detection/detectors/<detectorId>/_stop
```
#### Sample response
```json
updated requests and responses
2021-10-05 05:36:58 -04:00
{
"_id": "VEHKTXwBwf_U8gjUXY2s",
"_version": 0,
"_seq_no": 0,
"_primary_term": 0
}
incorporated feedback
2021-10-04 17:11:45 -04:00
```
To stop historical analysis:
```json
POST _plugins/_anomaly_detection/detectors/<detectorId>/_stop?historical=true
```
updated requests and responses
2021-10-05 05:36:58 -04:00
#### Sample response
```json
{
"_id": "f9DsTXwB6HknB84SoRTY",
"_version": 0,
"_seq_no": 0,
"_primary_term": 0
}
```
incorporated feedback
2021-10-04 17:11:45 -04:00
---
updated requests and responses
2021-10-05 05:36:58 -04:00
## Search detector
incorporated feedback
2021-10-04 17:11:45 -04:00
Introduced 1.0
{: .label .label-purple }
updated requests and responses
2021-10-05 05:36:58 -04:00
Returns all anomaly detectors for a search query.
To search detectors using the `server_log*` index:
incorporated feedback
2021-10-04 17:11:45 -04:00
#### Request
```json
updated requests and responses
2021-10-05 05:36:58 -04:00
GET _plugins/_anomaly_detection/detectors/_search
POST _plugins/_anomaly_detection/detectors/_search
incorporated feedback
2021-10-04 17:11:45 -04:00
{
"query": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"wildcard": {
"indices": {
"value": "server_log*"
incorporated feedback
2021-10-04 17:11:45 -04:00
}
}
}
}
```
#### Sample response
```json
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"took": 2,
incorporated feedback
2021-10-04 17:11:45 -04:00
"timed_out": false,
"_shards": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"total": 1,
"successful": 1,
incorporated feedback
2021-10-04 17:11:45 -04:00
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"value": 4,
incorporated feedback
2021-10-04 17:11:45 -04:00
"relation": "eq"
},
"max_score": 1,
"hits": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"_index": ".opendistro-anomaly-detectors",
incorporated feedback
2021-10-04 17:11:45 -04:00
"_type": "_doc",
updated requests and responses
2021-10-05 05:36:58 -04:00
"_id": "Zi5zTXwBwf_U8gjUTfJG",
incorporated feedback
2021-10-04 17:11:45 -04:00
"_version": 1,
updated requests and responses
2021-10-05 05:36:58 -04:00
"_seq_no": 1,
incorporated feedback
2021-10-04 17:11:45 -04:00
"_primary_term": 1,
"_score": 1,
"_source": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"name": "test",
"description": "test",
"time_field": "timestamp",
"indices": [
"server_log"
incorporated feedback
2021-10-04 17:11:45 -04:00
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"filter_query": {
"match_all": {
"boost": 1
}
},
"detection_interval": {
"period": {
"interval": 5,
"unit": "Minutes"
}
},
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"shingle_size": 8,
"schema_version": 0,
"feature_attributes": [
incorporated feedback
2021-10-04 17:11:45 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_id": "ZS5zTXwBwf_U8gjUTfIn",
"feature_name": "test_feature",
"feature_enabled": true,
"aggregation_query": {
"test_feature": {
"sum": {
"field": "value"
}
}
}
incorporated feedback
2021-10-04 17:11:45 -04:00
}
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"last_update_time": 1633386974533,
"category_field": [
"error_type"
],
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
"own_index",
"all_access"
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
},
"detector_type": "MULTI_ENTITY"
incorporated feedback
2021-10-04 17:11:45 -04:00
}
updated requests and responses
2021-10-05 05:36:58 -04:00
},
...
incorporated feedback
2021-10-04 17:11:45 -04:00
]
}
}
```
updated requests and responses
2021-10-05 05:36:58 -04:00
---
## Search detector tasks
Introduced 1.1
{: .label .label-purple }
Searches detector tasks.
incorporated feedback
2021-10-04 17:11:45 -04:00
incorporated feedback
2021-10-05 17:43:31 -04:00
To search for the latest detector level historical analysis task for a high cardinality detector
incorporated feedback
2021-10-04 17:11:45 -04:00
#### Request
```json
updated requests and responses
2021-10-05 05:36:58 -04:00
GET _plugins/_anomaly_detection/detectors/tasks/_search
POST _plugins/_anomaly_detection/detectors/tasks/_search
incorporated feedback
2021-10-04 17:11:45 -04:00
{
"query": {
"bool": {
"filter": [
{
"term": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"detector_id": "Zi5zTXwBwf_U8gjUTfJG"
incorporated feedback
2021-10-04 17:11:45 -04:00
}
},
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"term": {
"task_type": "HISTORICAL_HC_DETECTOR"
incorporated feedback
2021-10-04 17:11:45 -04:00
}
},
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"term": {
"is_latest": "true"
Breaks site
2021-05-28 13:48:19 -04:00
}
}
]
}
}
}
```
#### Sample response
```json
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"max_score": 0,
Breaks site
2021-05-28 13:48:19 -04:00
"hits": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"_index": ".opendistro-anomaly-detection-state",
Breaks site
2021-05-28 13:48:19 -04:00
"_type": "_doc",
updated requests and responses
2021-10-05 05:36:58 -04:00
"_id": "fm-RTXwBYwCbWecgB753",
"_version": 34,
"_seq_no": 928,
"_primary_term": 1,
"_score": 0,
Breaks site
2021-05-28 13:48:19 -04:00
"_source": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"detector_id": "Zi5zTXwBwf_U8gjUTfJG",
"error": "",
"detection_date_range": {
"start_time": 1630794960000,
"end_time": 1633386960000
},
"task_progress": 1,
"last_update_time": 1633389090738,
"execution_start_time": 1633388922742,
"state": "FINISHED",
"coordinating_node": "2Z4q22BySEyzakYt_A0A2A",
"task_type": "HISTORICAL_HC_DETECTOR",
"execution_end_time": 1633389090738,
Breaks site
2021-05-28 13:48:19 -04:00
"started_by": "admin",
updated requests and responses
2021-10-05 05:36:58 -04:00
"init_progress": 0,
Breaks site
2021-05-28 13:48:19 -04:00
"is_latest": true,
"detector": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"category_field": [
"error_type"
],
Breaks site
2021-05-28 13:48:19 -04:00
"description": "test",
"ui_metadata": {
"features": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"test_feature": {
Breaks site
2021-05-28 13:48:19 -04:00
"aggregationBy": "sum",
"aggregationOf": "value",
"featureType": "simple_aggs"
}
updated requests and responses
2021-10-05 05:36:58 -04:00
},
"filters": []
Breaks site
2021-05-28 13:48:19 -04:00
},
"feature_attributes": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_id": "ZS5zTXwBwf_U8gjUTfIn",
Breaks site
2021-05-28 13:48:19 -04:00
"feature_enabled": true,
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_name": "test_feature",
Breaks site
2021-05-28 13:48:19 -04:00
"aggregation_query": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"test_feature": {
Breaks site
2021-05-28 13:48:19 -04:00
"sum": {
"field": "value"
}
}
}
}
],
"schema_version": 0,
"time_field": "timestamp",
updated requests and responses
2021-10-05 05:36:58 -04:00
"last_update_time": 1633386974533,
Breaks site
2021-05-28 13:48:19 -04:00
"indices": [
more updates
2021-10-03 13:35:18 -04:00
"server_log"
Breaks site
2021-05-28 13:48:19 -04:00
],
"window_delay": {
"period": {
"unit": "Minutes",
"interval": 1
}
},
"detection_interval": {
"period": {
"unit": "Minutes",
updated requests and responses
2021-10-05 05:36:58 -04:00
"interval": 5
Breaks site
2021-05-28 13:48:19 -04:00
}
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"name": "testhc",
Breaks site
2021-05-28 13:48:19 -04:00
"filter_query": {
"match_all": {
"boost": 1
}
},
"shingle_size": 8,
"user": {
"backend_roles": [
"admin"
],
"custom_attribute_names": [],
"roles": [
updated requests and responses
2021-10-05 05:36:58 -04:00
"own_index",
"all_access"
Breaks site
2021-05-28 13:48:19 -04:00
],
"name": "admin",
"user_requested_tenant": "__user__"
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"detector_type": "MULTI_ENTITY"
Breaks site
2021-05-28 13:48:19 -04:00
},
"user": {
"backend_roles": [
"admin"
],
"custom_attribute_names": [],
"roles": [
updated requests and responses
2021-10-05 05:36:58 -04:00
"own_index",
"all_access"
Breaks site
2021-05-28 13:48:19 -04:00
],
"name": "admin",
"user_requested_tenant": "__user__"
}
}
}
]
}
}
```
incorporated feedback
2021-10-05 17:43:31 -04:00
To search for the latest entity-level tasks for the historical analysis of a high cardinality detector:
more updates
2021-10-03 13:35:18 -04:00
#### Request
```json
updated requests and responses
2021-10-05 05:36:58 -04:00
GET _plugins/_anomaly_detection/detectors/tasks/_search
more updates
2021-10-03 13:35:18 -04:00
POST _plugins/_anomaly_detection/detectors/tasks/_search
{
"query": {
"bool": {
"filter": [
{
"term": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"detector_id": "Zi5zTXwBwf_U8gjUTfJG"
more updates
2021-10-03 13:35:18 -04:00
}
},
{
"term": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"task_type": "HISTORICAL_HC_ENTITY"
}
},
{
"term": {
"is_latest": "true"
more updates
2021-10-03 13:35:18 -04:00
}
}
]
}
},
"sort": [
{
"execution_start_time": {
"order": "desc"
}
}
updated requests and responses
2021-10-05 05:36:58 -04:00
],
"size": 100
more updates
2021-10-03 13:35:18 -04:00
}
```
incorporated feedback
2021-10-05 17:43:31 -04:00
To search and aggregate states for all entity-level historical tasks:
The `parent_task_id` is the same as the task ID that you can get with the profile detector API:
`GET _plugins/_anomaly_detection/detectors/<detector_ID>/_profile/ad_task`.
{: .note }
updated requests and responses
2021-10-05 05:36:58 -04:00
#### Request
```json
GET _plugins/_anomaly_detection/detectors/tasks/_search
POST _plugins/_anomaly_detection/detectors/tasks/_search
{
"size": 0,
"query": {
"bool": {
"filter": [
{
"term": {
"detector_id": {
"value": "Zi5zTXwBwf_U8gjUTfJG",
"boost": 1
}
}
},
{
"term": {
"parent_task_id": {
"value": "fm-RTXwBYwCbWecgB753",
"boost": 1
}
}
},
{
"terms": {
"task_type": [
"HISTORICAL_HC_ENTITY"
],
"boost": 1
}
}
]
}
},
"aggs": {
"test": {
"terms": {
"field": "state",
"size": 100
}
}
}
}
```
more updates
2021-10-03 13:35:18 -04:00
#### Sample response
```json
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"took": 2,
ad changes for 1.1
2021-10-04 15:08:00 -04:00
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
more updates
2021-10-03 13:35:18 -04:00
},
ad changes for 1.1
2021-10-04 15:08:00 -04:00
"hits": {
"total": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"value": 32,
ad changes for 1.1
2021-10-04 15:08:00 -04:00
"relation": "eq"
more updates
2021-10-03 13:35:18 -04:00
},
ad changes for 1.1
2021-10-04 15:08:00 -04:00
"max_score": null,
updated requests and responses
2021-10-05 05:36:58 -04:00
"hits": []
},
"aggregations": {
"test": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "FINISHED",
"doc_count": 32
}
]
}
ad changes for 1.1
2021-10-04 15:08:00 -04:00
}
Breaks site
2021-05-28 13:48:19 -04:00
}
```
incorporated feedback
2021-10-04 17:11:45 -04:00
---
updated requests and responses
2021-10-05 05:36:58 -04:00
## Search detector result
incorporated feedback
2021-10-04 17:11:45 -04:00
Introduced 1.0
{: .label .label-purple }
updated requests and responses
2021-10-05 05:36:58 -04:00
Returns all results for a search query.
incorporated feedback
2021-10-04 17:11:45 -04:00
updated requests and responses
2021-10-05 05:36:58 -04:00
To search anomaly results for `grade` greater than 0 for real-time analysis:
incorporated feedback
2021-10-04 17:11:45 -04:00
#### Request
```json
updated requests and responses
2021-10-05 05:36:58 -04:00
GET _plugins/_anomaly_detection/detectors/results/_search
POST _plugins/_anomaly_detection/detectors/results/_search
incorporated feedback
2021-10-04 17:11:45 -04:00
{
"query": {
"bool": {
"filter": [
{
"term": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"detector_id": "Zi5zTXwBwf_U8gjUTfJG"
Breaks site
2021-05-28 13:48:19 -04:00
}
incorporated feedback
2021-10-04 17:11:45 -04:00
},
Breaks site
2021-05-28 13:48:19 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"range": {
"anomaly_grade": {
"gt": 0
Breaks site
2021-05-28 13:48:19 -04:00
}
}
updated requests and responses
2021-10-05 05:36:58 -04:00
}
],
"must_not": [
incorporated feedback
2021-10-04 17:11:45 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"exists": {
"field": "task_id"
Breaks site
2021-05-28 13:48:19 -04:00
}
}
incorporated feedback
2021-10-04 17:11:45 -04:00
]
Breaks site
2021-05-28 13:48:19 -04:00
}
}
}
```
incorporated feedback
2021-10-05 17:43:31 -04:00
Real-time detection doesn't persist the task ID in the anomaly result, so the task ID will be null.
incorporated feedback
2021-10-04 17:11:45 -04:00
#### Sample response
```json
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"took": 4,
incorporated feedback
2021-10-04 17:11:45 -04:00
"timed_out": false,
updated requests and responses
2021-10-05 05:36:58 -04:00
"_shards": {
"total": 3,
"successful": 3,
"skipped": 0,
"failed": 0
incorporated feedback
2021-10-04 17:11:45 -04:00
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"hits": {
"total": {
"value": 90,
"relation": "eq"
},
"max_score": 0,
"hits": [
{
"_index": ".opendistro-anomaly-results-history-2021.10.04-1",
"_type": "_doc",
"_id": "686KTXwB6HknB84SMr6G",
"_version": 1,
"_seq_no": 103622,
"_primary_term": 1,
"_score": 0,
"_source": {
"detector_id": "Zi5zTXwBwf_U8gjUTfJG",
"confidence": 0.918886275269358,
"model_id": "Zi5zTXwBwf_U8gjUTfJG_entity_error16",
"schema_version": 4,
"anomaly_score": 1.1093755891885446,
"execution_start_time": 1633388475001,
"data_end_time": 1633388414989,
"data_start_time": 1633388114989,
"feature_data": [
{
"feature_id": "ZS5zTXwBwf_U8gjUTfIn",
"feature_name": "test_feature",
"data": 0.532
}
],
"execution_end_time": 1633388475014,
"user": {
"backend_roles": [
"admin"
],
"custom_attribute_names": [],
"roles": [
"own_index",
"all_access"
],
"name": "admin",
"user_requested_tenant": "__user__"
},
"anomaly_grade": 0.031023547546561225,
"entity": [
{
"name": "error_type",
"value": "error16"
}
]
}
},
...
]
}
incorporated feedback
2021-10-04 17:11:45 -04:00
}
```
updated requests and responses
2021-10-05 05:36:58 -04:00
You can run historical analysis as many times as you like. So, multiple tasks might exist for the same detector.
incorporated feedback
2021-10-04 17:11:45 -04:00
updated requests and responses
2021-10-05 05:36:58 -04:00
You can search for the latest historical batch task first and then search the historical batch task results.
Breaks site
2021-05-28 13:48:19 -04:00
updated requests and responses
2021-10-05 05:36:58 -04:00
To search anomaly results for `grade` greater than 0 for historical analysis with the `task_id`:
Breaks site
2021-05-28 13:48:19 -04:00
#### Request
```json
updated requests and responses
2021-10-05 05:36:58 -04:00
GET _plugins/_anomaly_detection/detectors/results/_search
POST _plugins/_anomaly_detection/detectors/results/_search
Breaks site
2021-05-28 13:48:19 -04:00
{
"query": {
more updates
2021-10-03 13:35:18 -04:00
"bool": {
"filter": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"term": {
"detector_id": "Zi5zTXwBwf_U8gjUTfJG"
}
},
{
"range": {
"anomaly_grade": {
"gt": 0
}
}
},
{
"term": {
"task_id": "fm-RTXwBYwCbWecgB753"
more updates
2021-10-03 13:35:18 -04:00
}
}
]
Breaks site
2021-05-28 13:48:19 -04:00
}
}
}
```
#### Sample response
```json
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"took": 915,
Breaks site
2021-05-28 13:48:19 -04:00
"timed_out": false,
"_shards": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"total": 3,
"successful": 3,
Breaks site
2021-05-28 13:48:19 -04:00
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"value": 4115,
Breaks site
2021-05-28 13:48:19 -04:00
"relation": "eq"
},
ad changes for 1.1
2021-10-04 15:08:00 -04:00
"max_score": 0,
Breaks site
2021-05-28 13:48:19 -04:00
"hits": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"_index": ".opendistro-anomaly-results-history-2021.10.04-1",
Breaks site
2021-05-28 13:48:19 -04:00
"_type": "_doc",
updated requests and responses
2021-10-05 05:36:58 -04:00
"_id": "VRyRTXwBDx7vzPBV8jYC",
ad changes for 1.1
2021-10-04 15:08:00 -04:00
"_version": 1,
updated requests and responses
2021-10-05 05:36:58 -04:00
"_seq_no": 149657,
Breaks site
2021-05-28 13:48:19 -04:00
"_primary_term": 1,
ad changes for 1.1
2021-10-04 15:08:00 -04:00
"_score": 0,
Breaks site
2021-05-28 13:48:19 -04:00
"_source": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"detector_id": "Zi5zTXwBwf_U8gjUTfJG",
"confidence": 0.9642989263957601,
"task_id": "fm-RTXwBYwCbWecgB753",
"model_id": "Zi5zTXwBwf_U8gjUTfJG_entity_error24",
"schema_version": 4,
"anomaly_score": 1.2260712437521946,
"execution_start_time": 1633388982692,
"data_end_time": 1631721300000,
"data_start_time": 1631721000000,
"feature_data": [
Breaks site
2021-05-28 13:48:19 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_id": "ZS5zTXwBwf_U8gjUTfIn",
"feature_name": "test_feature",
"data": 10
Breaks site
2021-05-28 13:48:19 -04:00
}
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"execution_end_time": 1633388982709,
ad changes for 1.1
2021-10-04 15:08:00 -04:00
"user": {
"backend_roles": [
"admin"
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"custom_attribute_names": [],
ad changes for 1.1
2021-10-04 15:08:00 -04:00
"roles": [
"own_index",
"all_access"
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"name": "admin",
"user_requested_tenant": "__user__"
ad changes for 1.1
2021-10-04 15:08:00 -04:00
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"anomaly_grade": 0.14249628345655782,
"entity": [
{
"name": "error_type",
"value": "error1"
}
]
Breaks site
2021-05-28 13:48:19 -04:00
}
updated requests and responses
2021-10-05 05:36:58 -04:00
},
...
Breaks site
2021-05-28 13:48:19 -04:00
]
}
}
```
---
## Get detector stats
Add "introduced" labels for APIs
2021-07-26 19:14:22 -04:00
Introduced 1.0
{: .label .label-purple }
Breaks site
2021-05-28 13:48:19 -04:00
Provides information about how the plugin is performing.
updated requests and responses
2021-10-05 05:36:58 -04:00
To get all stats:
Breaks site
2021-05-28 13:48:19 -04:00
#### Request
```json
GET _plugins/_anomaly_detection/stats
```
#### Sample response
```json
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"anomaly_detectors_index_status": "green",
"anomaly_detection_state_status": "green",
"single_entity_detector_count": 2,
"detector_count": 5,
"multi_entity_detector_count": 3,
"anomaly_detection_job_index_status": "green",
"models_checkpoint_index_status": "green",
"anomaly_results_index_status": "green",
Breaks site
2021-05-28 13:48:19 -04:00
"nodes": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"2Z4q22BySEyzakYt_A0A2A": {
"ad_execute_request_count": 95,
more updates
2021-10-03 13:35:18 -04:00
"models": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"detector_id": "WTBnTXwBjd8s6RK4b1Sz",
"model_type": "rcf",
"last_used_time": 1633398197185,
"model_id": "WTBnTXwBjd8s6RK4b1Sz_model_rcf_0",
"last_checkpoint_time": 1633396573679
more updates
2021-10-03 13:35:18 -04:00
},
updated requests and responses
2021-10-05 05:36:58 -04:00
...
],
"ad_canceled_batch_task_count": 0,
"ad_hc_execute_request_count": 75,
"ad_hc_execute_failure_count": 0,
"model_count": 28,
"ad_execute_failure_count": 1,
"ad_batch_task_failure_count": 0,
"ad_total_batch_task_execution_count": 27,
"ad_executing_batch_task_count": 3
},
"SWD7ihu9TaaW1zKwFZNVNg": {
"ad_execute_request_count": 12,
"models": [
more updates
2021-10-03 13:35:18 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"detector_id": "Zi5zTXwBwf_U8gjUTfJG",
more updates
2021-10-03 13:35:18 -04:00
"model_type": "entity",
updated requests and responses
2021-10-05 05:36:58 -04:00
"last_used_time": 1633398375008,
"model_id": "Zi5zTXwBwf_U8gjUTfJG_entity_error13",
"last_checkpoint_time": 1633392973682,
more updates
2021-10-03 13:35:18 -04:00
"entity": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"name": "error_type",
"value": "error13"
more updates
2021-10-03 13:35:18 -04:00
}
]
},
updated requests and responses
2021-10-05 05:36:58 -04:00
...
],
"ad_canceled_batch_task_count": 1,
"ad_hc_execute_request_count": 0,
"ad_hc_execute_failure_count": 0,
"model_count": 15,
"ad_execute_failure_count": 2,
"ad_batch_task_failure_count": 0,
"ad_total_batch_task_execution_count": 27,
"ad_executing_batch_task_count": 4
},
"TQDUXEzyTJyV0H6_T4hYUw": {
"ad_execute_request_count": 0,
"models": [
more updates
2021-10-03 13:35:18 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"detector_id": "Zi5zTXwBwf_U8gjUTfJG",
more updates
2021-10-03 13:35:18 -04:00
"model_type": "entity",
updated requests and responses
2021-10-05 05:36:58 -04:00
"last_used_time": 1633398375004,
"model_id": "Zi5zTXwBwf_U8gjUTfJG_entity_error24",
"last_checkpoint_time": 1633388177359,
more updates
2021-10-03 13:35:18 -04:00
"entity": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"name": "error_type",
"value": "error24"
more updates
2021-10-03 13:35:18 -04:00
}
]
},
updated requests and responses
2021-10-05 05:36:58 -04:00
...
more updates
2021-10-03 13:35:18 -04:00
],
"ad_canceled_batch_task_count": 0,
updated requests and responses
2021-10-05 05:36:58 -04:00
"ad_hc_execute_request_count": 0,
more updates
2021-10-03 13:35:18 -04:00
"ad_hc_execute_failure_count": 0,
updated requests and responses
2021-10-05 05:36:58 -04:00
"model_count": 22,
more updates
2021-10-03 13:35:18 -04:00
"ad_execute_failure_count": 0,
"ad_batch_task_failure_count": 0,
updated requests and responses
2021-10-05 05:36:58 -04:00
"ad_total_batch_task_execution_count": 28,
"ad_executing_batch_task_count": 3
more updates
2021-10-03 13:35:18 -04:00
}
}
}
```
The `model_count` parameter shows the total number of models running on each node’s memory.
historical detector to historical analysis
2021-10-04 15:38:02 -04:00
For historical analysis, you see the values for the following fields:
more updates
2021-10-03 13:35:18 -04:00
- `ad_total_batch_task_execution_count`
- `ad_executing_batch_task_count`
- `ad_canceled_batch_task_count`
- `ad_batch_task_failure_count`
incorporated feedback
2021-10-04 17:11:45 -04:00
If haven't run any historical analysis, these values show up as 0.
historical detector to historical analysis
2021-10-04 15:38:02 -04:00
updated requests and responses
2021-10-05 05:36:58 -04:00
To get all stats for a specific node:
#### Request
```json
GET _plugins/_anomaly_detection/<nodeId>/stats
```
To get specific stats for a node:
#### Request
```json
GET _plugins/_anomaly_detection/<nodeId>/stats/<stat>
incorporated feedback
2021-10-05 17:43:31 -04:00
```
For example:
```json
updated requests and responses
2021-10-05 05:36:58 -04:00
GET _plugins/_anomaly_detection/<nodeId>/stats/ad_execute_request_count
```
more updates
2021-10-03 13:35:18 -04:00
#### Sample response
```json
{
"nodes": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"SWD7ihu9TaaW1zKwFZNVNg": {
"ad_execute_request_count": 12
}
}
}
```
To get a specific type of stats:
#### Request
```json
GET _plugins/_anomaly_detection/stats/<stat>
incorporated feedback
2021-10-05 17:43:31 -04:00
```
For example:
```json
updated requests and responses
2021-10-05 05:36:58 -04:00
GET _plugins/_anomaly_detection/stats/ad_executing_batch_task_count
```
#### Sample response
```json
{
"nodes": {
"2Z4q22BySEyzakYt_A0A2A": {
ad changes for 1.1
2021-10-04 15:08:00 -04:00
"ad_executing_batch_task_count": 3
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"SWD7ihu9TaaW1zKwFZNVNg": {
more updates
2021-10-03 13:35:18 -04:00
"ad_executing_batch_task_count": 3
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"TQDUXEzyTJyV0H6_T4hYUw": {
"ad_executing_batch_task_count": 4
more updates
2021-10-03 13:35:18 -04:00
}
}
}
```
---
updated requests and responses
2021-10-05 05:36:58 -04:00
## Profile detector
more updates
2021-10-03 13:35:18 -04:00
Introduced 1.0
{: .label .label-purple }
updated requests and responses
2021-10-05 05:36:58 -04:00
Returns information related to the current state of the detector and memory usage, including current errors and shingle size, to help troubleshoot the detector.
This command helps locate logs by identifying the nodes that run the anomaly detector job for each detector.
It also helps track the initialization percentage, the required shingles, and the estimated time left.
more updates
2021-10-03 13:35:18 -04:00
#### Request
```json
updated requests and responses
2021-10-05 05:36:58 -04:00
GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile/
GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile?_all=true
GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile/<type>
GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile/<type1>,<type2>
```
#### Sample Responses
```json
GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile
more updates
2021-10-03 13:35:18 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"state": "DISABLED",
"error": "Stopped detector: AD models memory usage exceeds our limit."
}
GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile?_all=true&pretty
{
"state": "RUNNING",
"error": "",
"models": [
more updates
2021-10-03 13:35:18 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"model_id": "3Dh6TXwBwf_U8gjURE0F_entity_KSLSh0Wv05RQXiBAQHTEZg",
more updates
2021-10-03 13:35:18 -04:00
"entity": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"name": "ip",
"value": "192.168.1.1"
ad changes for 1.1
2021-10-04 15:08:00 -04:00
},
more updates
2021-10-03 13:35:18 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"name": "error_type",
"value": "error8"
more updates
2021-10-03 13:35:18 -04:00
}
],
"model_size_in_bytes": 403491,
updated requests and responses
2021-10-05 05:36:58 -04:00
"node_id": "2Z4q22BySEyzakYt_A0A2A"
Breaks site
2021-05-28 13:48:19 -04:00
},
updated requests and responses
2021-10-05 05:36:58 -04:00
...
Breaks site
2021-05-28 13:48:19 -04:00
],
more updates
2021-10-03 13:35:18 -04:00
"total_size_in_bytes": 12911712,
Breaks site
2021-05-28 13:48:19 -04:00
"init_progress": {
more updates
2021-10-03 13:35:18 -04:00
"percentage": "100%"
},
"total_entities": 33,
"active_entities": 32,
"ad_task": {
"ad_task": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"task_id": "D3I5TnwBYwCbWecg7lN9",
"last_update_time": 1633399993685,
more updates
2021-10-03 13:35:18 -04:00
"started_by": "admin",
"state": "RUNNING",
updated requests and responses
2021-10-05 05:36:58 -04:00
"detector_id": "3Dh6TXwBwf_U8gjURE0F",
more updates
2021-10-03 13:35:18 -04:00
"task_progress": 0,
"init_progress": 0,
updated requests and responses
2021-10-05 05:36:58 -04:00
"execution_start_time": 1633399991933,
more updates
2021-10-03 13:35:18 -04:00
"is_latest": true,
"task_type": "HISTORICAL_HC_DETECTOR",
updated requests and responses
2021-10-05 05:36:58 -04:00
"coordinating_node": "2Z4q22BySEyzakYt_A0A2A",
more updates
2021-10-03 13:35:18 -04:00
"detector": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"name": "testhc-mc",
more updates
2021-10-03 13:35:18 -04:00
"description": "test",
"time_field": "timestamp",
"indices": [
"server_log"
],
"filter_query": {
"match_all": {
"boost": 1
}
},
"detection_interval": {
"period": {
"interval": 5,
"unit": "Minutes"
}
},
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"shingle_size": 8,
"schema_version": 0,
"feature_attributes": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"feature_id": "2zh6TXwBwf_U8gjUQ039",
"feature_name": "test",
more updates
2021-10-03 13:35:18 -04:00
"feature_enabled": true,
"aggregation_query": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"test": {
more updates
2021-10-03 13:35:18 -04:00
"sum": {
"field": "value"
}
}
}
}
],
"ui_metadata": {
"features": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"test": {
more updates
2021-10-03 13:35:18 -04:00
"aggregationBy": "sum",
"aggregationOf": "value",
"featureType": "simple_aggs"
}
},
"filters": []
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"last_update_time": 1633387430916,
more updates
2021-10-03 13:35:18 -04:00
"category_field": [
updated requests and responses
2021-10-05 05:36:58 -04:00
"ip",
"error_type"
more updates
2021-10-03 13:35:18 -04:00
],
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
"own_index",
"all_access"
],
"custom_attribute_names": [],
updated requests and responses
2021-10-05 05:36:58 -04:00
"user_requested_tenant": "__user__"
more updates
2021-10-03 13:35:18 -04:00
},
"detector_type": "MULTI_ENTITY"
},
"detection_date_range": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"start_time": 1632793800000,
"end_time": 1633398600000
more updates
2021-10-03 13:35:18 -04:00
},
"user": {
"name": "admin",
updated requests and responses
2021-10-05 05:36:58 -04:00
"backend_roles": [
"admin"
more updates
2021-10-03 13:35:18 -04:00
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"roles": [
"own_index",
"all_access"
more updates
2021-10-03 13:35:18 -04:00
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
}
},
"node_id": "2Z4q22BySEyzakYt_A0A2A",
"task_id": "D3I5TnwBYwCbWecg7lN9",
"task_type": "HISTORICAL_HC_DETECTOR",
"detector_task_slots": 10,
"total_entities_count": 32,
"pending_entities_count": 22,
"running_entities_count": 10,
"running_entities": [ """[{"name":"ip","value":"192.168.1.1"},{"name":"error_type","value":"error9"}]""",
...],
"entity_task_profiles": [
more updates
2021-10-03 13:35:18 -04:00
{
"shingle_size": 8,
updated requests and responses
2021-10-05 05:36:58 -04:00
"rcf_total_updates": 1994,
more updates
2021-10-03 13:35:18 -04:00
"threshold_model_trained": true,
"threshold_model_training_data_size": 0,
"model_size_in_bytes": 1593240,
updated requests and responses
2021-10-05 05:36:58 -04:00
"node_id": "2Z4q22BySEyzakYt_A0A2A",
more updates
2021-10-03 13:35:18 -04:00
"entity": [
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"name": "ip",
"value": "192.168.1.1"
},
more updates
2021-10-03 13:35:18 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"name": "error_type",
more updates
2021-10-03 13:35:18 -04:00
"value": "error7"
}
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"task_id": "E3I5TnwBYwCbWecg9FMm",
more updates
2021-10-03 13:35:18 -04:00
"task_type": "HISTORICAL_HC_ENTITY"
updated requests and responses
2021-10-05 05:36:58 -04:00
},
...
more updates
2021-10-03 13:35:18 -04:00
]
},
"model_count": 32
Breaks site
2021-05-28 13:48:19 -04:00
}
updated requests and responses
2021-10-05 05:36:58 -04:00
GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile/total_size_in_bytes
{
"total_size_in_bytes": 13369344
}
Breaks site
2021-05-28 13:48:19 -04:00
```
updated requests and responses
2021-10-05 05:36:58 -04:00
You can see the `ad_task` field only for historical analysis.
The `model_count` parameter shows the total number of models that a detector runs on each node’s memory. This is useful if you have several models running on your cluster and want to know the count.
If you configured the category field, you can see the number of unique values in the field and all active entities with models running in memory.
You can use this data to estimate how much memory is required for anomaly detection so you can decide how to size your cluster. For example, if a detector has one million entities and only 10 of them are active in memory, you need to scale your cluster up or out.
ad changes for 1.1
2021-10-04 15:08:00 -04:00
For a single-entity detector:
Breaks site
2021-05-28 13:48:19 -04:00
ad changes for 1.1
2021-10-04 15:08:00 -04:00
#### Sample response
Breaks site
2021-05-28 13:48:19 -04:00
```json
{
ad changes for 1.1
2021-10-04 15:08:00 -04:00
"state": "INIT",
"total_size_in_bytes": 0,
Breaks site
2021-05-28 13:48:19 -04:00
"init_progress": {
ad changes for 1.1
2021-10-04 15:08:00 -04:00
"percentage": "0%",
"needed_shingles": 128
Breaks site
2021-05-28 13:48:19 -04:00
},
ad changes for 1.1
2021-10-04 15:08:00 -04:00
"ad_task": {
"ad_task": {
"task_id": "cfUNOXwBFLNqSEcxAlde",
"last_update_time": 1633044731640,
"started_by": "admin",
"state": "RUNNING",
"detector_id": "qL4NOXwB__6eNorTAKtJ",
"task_progress": 0.49603173,
"init_progress": 1,
"current_piece": 1632739800000,
"execution_start_time": 1633044726365,
"is_latest": true,
"task_type": "HISTORICAL_SINGLE_ENTITY",
"coordinating_node": "bCtWtxWPThq0BIn5P5I4Xw",
"worker_node": "dIyavWhmSYWGz65b4u-lpQ",
"detector": {
"name": "detector1",
"description": "test",
"time_field": "timestamp",
"indices": [
"server_log"
],
"filter_query": {
"match_all": {
"boost": 1
}
},
"detection_interval": {
"period": {
"interval": 5,
"unit": "Minutes"
}
},
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"shingle_size": 8,
"schema_version": 0,
"feature_attributes": [
{
"feature_id": "p74NOXwB__6eNorTAKss",
"feature_name": "test-feature",
"feature_enabled": true,
"aggregation_query": {
"test_feature": {
"sum": {
"field": "value"
}
}
}
}
],
"ui_metadata": {
"features": {
"test-feature": {
"aggregationBy": "sum",
"aggregationOf": "value",
"featureType": "simple_aggs"
}
},
"filters": []
},
"last_update_time": 1633044725832,
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
"own_index",
"all_access"
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
},
"detector_type": "SINGLE_ENTITY"
},
"detection_date_range": {
"start_time": 1632439925885,
"end_time": 1633044725885
},
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
"own_index",
"all_access"
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
}
},
"shingle_size": 8,
"rcf_total_updates": 1994,
"threshold_model_trained": true,
"threshold_model_training_data_size": 0,
"model_size_in_bytes": 1593240,
"node_id": "dIyavWhmSYWGz65b4u-lpQ",
"detector_task_slots": 1
}
Breaks site
2021-05-28 13:48:19 -04:00
}
```
more updates
2021-10-03 13:35:18 -04:00
The `total_entities` parameter shows you the total number of entities including the number of category fields for a detector.
updated requests and responses
2021-10-05 05:36:58 -04:00
Getting the total count of entities is an expensive operation for real-time analysis of a detector with more than one category field. By default, for a real-time detection profile, a detector counts the number of entities up to a value of 10,000. For historical analysis, the anomaly detection plugin only detects the top 1,000 entities by default and caches the top entities in memory, so it doesn't cost much to get the total count of entities for historical analysis.
more updates
2021-10-03 13:35:18 -04:00
ad changes for 1.1
2021-10-04 15:08:00 -04:00
The `profile` operation also provides information about each entity, such as the entity’s `last_sample_timestamp` and `last_active_timestamp`. `last_sample_timestamp` shows the last document in the input data source index containing the entity, while `last_active_timestamp` shows the timestamp when the entity’s model was last seen in the model cache.
Breaks site
2021-05-28 13:48:19 -04:00
incorporated feedback
2021-10-05 17:43:31 -04:00
If there are no anomaly results for an entity, either the entity doesn't have any sample data or resources such as memory and disk IO are constrained relative to the number of entities.
Breaks site
2021-05-28 13:48:19 -04:00
#### Request
```json
more updates
2021-10-03 13:35:18 -04:00
GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile?_all=true
{
"entity": [
{
"name": "host",
"value": "i-00f28ec1eb8997686"
}
]
}
```
#### Sample Responses
```json
Breaks site
2021-05-28 13:48:19 -04:00
{
"category_field": "host",
"value": "i-00f28ec1eb8997686",
"is_active": true,
"last_active_timestamp": 1604026394879,
"last_sample_timestamp": 1604026394879,
"init_progress": {
"percentage": "100%"
},
"model": {
"model_id": "TFUdd3UBBwIAGQeRh5IS_entity_i-00f28ec1eb8997686",
"model_size_in_bytes": 712480,
"node_id": "MQ-bTBW3Q2uU_2zX3pyEQg"
},
"state": "RUNNING"
}
```
updated requests and responses
2021-10-05 05:36:58 -04:00
To get profile information for only historical analysis, specify `ad_task`.
Specifying `_all` is an expensive operation for multi-category high cardinality detectors.
#### Request
```json
GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile?_all
GET _plugins/_anomaly_detection/detectors/<detectorId>/_profile/ad_task
```
#### Sample Responses
```json
{
"ad_task": {
"ad_task": {
"task_id": "CHI0TnwBYwCbWecgqgRA",
"last_update_time": 1633399648413,
"started_by": "admin",
"state": "RUNNING",
"detector_id": "3Dh6TXwBwf_U8gjURE0F",
"task_progress": 0,
"init_progress": 0,
"execution_start_time": 1633399646784,
"is_latest": true,
"task_type": "HISTORICAL_HC_DETECTOR",
"coordinating_node": "2Z4q22BySEyzakYt_A0A2A",
"detector": {
"name": "testhc-mc",
"description": "test",
"time_field": "timestamp",
"indices": [
"server_log"
],
"filter_query": {
"match_all": {
"boost": 1
}
},
"detection_interval": {
"period": {
"interval": 5,
"unit": "Minutes"
}
},
"window_delay": {
"period": {
"interval": 1,
"unit": "Minutes"
}
},
"shingle_size": 8,
"schema_version": 0,
"feature_attributes": [
{
"feature_id": "2zh6TXwBwf_U8gjUQ039",
"feature_name": "test",
"feature_enabled": true,
"aggregation_query": {
"test": {
"sum": {
"field": "value"
}
}
}
}
],
"ui_metadata": {
"features": {
"test": {
"aggregationBy": "sum",
"aggregationOf": "value",
"featureType": "simple_aggs"
}
},
"filters": []
},
"last_update_time": 1633387430916,
"category_field": [
"ip",
"error_type"
],
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
"own_index",
"all_access"
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
},
"detector_type": "MULTI_ENTITY"
},
"detection_date_range": {
"start_time": 1632793800000,
"end_time": 1633398600000
},
"user": {
"name": "admin",
"backend_roles": [
"admin"
],
"roles": [
"own_index",
"all_access"
],
"custom_attribute_names": [],
"user_requested_tenant": "__user__"
}
},
"node_id": "2Z4q22BySEyzakYt_A0A2A",
"task_id": "CHI0TnwBYwCbWecgqgRA",
"task_type": "HISTORICAL_HC_DETECTOR",
"detector_task_slots": 10,
"total_entities_count": 32,
"pending_entities_count": 22,
"running_entities_count": 10,
"running_entities" : [
"""[{"name":"ip","value":"192.168.1.1"},{"name":"error_type","value":"error9"}]""",
...
],
"entity_task_profiles": [
{
"shingle_size": 8,
"rcf_total_updates": 994,
"threshold_model_trained": true,
"threshold_model_training_data_size": 0,
"model_size_in_bytes": 1593240,
"node_id": "2Z4q22BySEyzakYt_A0A2A",
"entity": [
{
"name": "ip",
"value": "192.168.1.1"
},
{
"name": "error_type",
"value": "error6"
}
],
"task_id": "9XI0TnwBYwCbWecgsAd6",
"task_type": "HISTORICAL_HC_ENTITY"
},
...
]
}
}
```
---
## Delete detector results
Introduced 1.1
{: .label .label-purple }
Deletes the results of a detector based on a query.
#### Request
```json
DELETE _plugins/_anomaly_detection/detectors/results
{
"query": {
"bool": {
"filter": [
{
"term": {
"detector_id": {
"value": "rlDtOHwBD5tpxlbyW7Nt"
}
}
},
{
"term": {
"task_id": {
"value": "TM3tOHwBCi2h__AOXlyQ"
}
}
},
{
"range": {
"data_start_time": {
"lte": 1632441600000
}
}
}
]
}
}
}
```
#### Sample response
```json
{
"took": 48,
"timed_out": false,
"total": 28,
"updated": 0,
"created": 0,
"deleted": 28,
"batches": 1,
"version_conflicts": 0,
"noops": 0,
"retries": {
"bulk": 0,
"search": 0
},
"throttled_millis": 0,
"requests_per_second": -1,
"throttled_until_millis": 0,
"failures": []
}
```
---
## Create monitor
Introduced 1.0
{: .label .label-purple }
Create a monitor to set up alerts for the detector.
Breaks site
2021-05-28 13:48:19 -04:00
#### Request
```json
updated requests and responses
2021-10-05 05:36:58 -04:00
POST _plugins/_alerting/monitors
Breaks site
2021-05-28 13:48:19 -04:00
{
updated requests and responses
2021-10-05 05:36:58 -04:00
"type": "monitor",
"name": "test-monitor",
"enabled": true,
"schedule": {
"period": {
"interval": 20,
"unit": "MINUTES"
}
},
"inputs": [
{
"search": {
Breaks site
2021-05-28 13:48:19 -04:00
"indices": [
updated requests and responses
2021-10-05 05:36:58 -04:00
".opensearch-anomaly-results*"
Breaks site
2021-05-28 13:48:19 -04:00
],
updated requests and responses
2021-10-05 05:36:58 -04:00
"query": {
"size": 1,
"query": {
"bool": {
"filter": [
{
"range": {
"data_end_time": {
"from": "{{period_end}}||-20m",
"to": "{{period_end}}",
"include_lower": true,
"include_upper": true,
"boost": 1
}
}
},
{
"term": {
"detector_id": {
"value": "m4ccEnIBTXsGi3mvMt9p",
"boost": 1
}
}
Breaks site
2021-05-28 13:48:19 -04:00
}
updated requests and responses
2021-10-05 05:36:58 -04:00
],
"adjust_pure_negative": true,
"boost": 1
Breaks site
2021-05-28 13:48:19 -04:00
}
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"sort": [
{
"anomaly_grade": {
"order": "desc"
}
},
{
"confidence": {
"order": "desc"
}
}
],
"aggregations": {
"max_anomaly_grade": {
Breaks site
2021-05-28 13:48:19 -04:00
"max": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"field": "anomaly_grade"
}
}
}
}
}
}
],
"triggers": [
{
"name": "test-trigger",
"severity": "1",
"condition": {
"script": {
"source": "return ctx.results[0].aggregations.max_anomaly_grade.value != null && ctx.results[0].aggregations.max_anomaly_grade.value > 0.7 && ctx.results[0].hits.hits[0]._source.confidence > 0.7",
"lang": "painless"
}
},
"actions": [
{
"name": "test-action",
"destination_id": "ld7912sBlQ5JUWWFThoW",
"message_template": {
"source": "This is my message body."
},
"throttle_enabled": false,
"subject_template": {
"source": "TheSubject"
}
}
]
}
]
}
```
#### Sample response
```json
{
"_id": "OClTEnIBmSf7y6LP11Jz",
"_version": 1,
"_seq_no": 10,
"_primary_term": 1,
"monitor": {
"type": "monitor",
"schema_version": 1,
"name": "test-monitor",
"enabled": true,
"enabled_time": 1589445384043,
"schedule": {
"period": {
"interval": 20,
"unit": "MINUTES"
}
},
"inputs": [
{
"search": {
"indices": [
".opensearch-anomaly-results*"
],
"query": {
"size": 1,
"query": {
"bool": {
"filter": [
{
"range": {
"data_end_time": {
"from": "{{period_end}}||-20m",
"to": "{{period_end}}",
"include_lower": true,
"include_upper": true,
"boost": 1
}
}
},
{
"term": {
"detector_id": {
"value": "m4ccEnIBTXsGi3mvMt9p",
"boost": 1
}
}
}
],
"adjust_pure_negative": true,
"boost": 1
}
},
"sort": [
{
"anomaly_grade": {
"order": "desc"
}
},
{
"confidence": {
"order": "desc"
}
}
],
"aggregations": {
"max_anomaly_grade": {
Breaks site
2021-05-28 13:48:19 -04:00
"max": {
updated requests and responses
2021-10-05 05:36:58 -04:00
"field": "anomaly_grade"
Breaks site
2021-05-28 13:48:19 -04:00
}
}
}
}
updated requests and responses
2021-10-05 05:36:58 -04:00
}
}
],
"triggers": [
{
"id": "NilTEnIBmSf7y6LP11Jr",
"name": "test-trigger",
"severity": "1",
"condition": {
"script": {
"source": "return ctx.results[0].aggregations.max_anomaly_grade.value != null && ctx.results[0].aggregations.max_anomaly_grade.value > 0.7 && ctx.results[0].hits.hits[0]._source.confidence > 0.7",
"lang": "painless"
}
},
"actions": [
{
"id": "NylTEnIBmSf7y6LP11Jr",
"name": "test-action",
"destination_id": "ld7912sBlQ5JUWWFThoW",
"message_template": {
"source": "This is my message body.",
"lang": "mustache"
Breaks site
2021-05-28 13:48:19 -04:00
},
updated requests and responses
2021-10-05 05:36:58 -04:00
"throttle_enabled": false,
"subject_template": {
"source": "TheSubject",
"lang": "mustache"
Breaks site
2021-05-28 13:48:19 -04:00
}
updated requests and responses
2021-10-05 05:36:58 -04:00
}
]
Breaks site
2021-05-28 13:48:19 -04:00
}
updated requests and responses
2021-10-05 05:36:58 -04:00
],
"last_update_time": 1589445384043
Breaks site
2021-05-28 13:48:19 -04:00
}
}
```
---