From a23cdb91aeced6df567216c70143b8c880e48c6e Mon Sep 17 00:00:00 2001 From: ashwinkumar12345 Date: Tue, 5 Oct 2021 02:36:58 -0700 Subject: [PATCH] updated requests and responses --- _monitoring-plugins/ad/api.md | 4237 +++++++++++++--------------- _monitoring-plugins/ad/index.md | 2 +- _monitoring-plugins/ad/settings.md | 2 +- 3 files changed, 2032 insertions(+), 2209 deletions(-) diff --git a/_monitoring-plugins/ad/api.md b/_monitoring-plugins/ad/api.md index f99825fd..46c12c28 100644 --- a/_monitoring-plugins/ad/api.md +++ b/_monitoring-plugins/ad/api.md @@ -24,8 +24,7 @@ Introduced 1.0 Creates an anomaly detector. -This command creates a detector named `test-detector` that finds anomalies based on the sum of the `value` field: - +This command creates a single-flow detector named `test-detector` that finds anomalies based on the sum of the `value` field: #### Request @@ -36,14 +35,14 @@ POST _plugins/_anomaly_detection/detectors "description": "Test detector", "time_field": "timestamp", "indices": [ - "order*" + "server_log*" ], "feature_attributes": [ { - "feature_name": "total_order", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "total_order": { + "test": { "sum": { "field": "value" } @@ -55,9 +54,10 @@ POST _plugins/_anomaly_detection/detectors "bool": { "filter": [ { - "exists": { - "field": "value", - "boost": 1 + "range": { + "value": { + "gt": 1 + } } } ], @@ -84,24 +84,28 @@ POST _plugins/_anomaly_detection/detectors ```json { - "_id": "m4ccEnIBTXsGi3mvMt9p", + "_id": "VEHKTXwBwf_U8gjUXY2s", "_version": 1, - "_seq_no": 3, - "_primary_term": 1, + "_seq_no": 5, "anomaly_detector": { "name": "test-detector", "description": "Test detector", "time_field": "timestamp", "indices": [ - "order*" + "server_log*" ], "filter_query": { "bool": { "filter": [ { - "exists": { - "field": "value", - "boost": 1 + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } } } ], @@ -121,66 +125,97 @@ POST _plugins/_anomaly_detection/detectors "unit": "Minutes" } }, + "shingle_size": 8, "schema_version": 0, "feature_attributes": [ { - "feature_id": "mYccEnIBTXsGi3mvMd8_", - "feature_name": "total_order", + "feature_id": "U0HKTXwBwf_U8gjUXY2m", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "total_order": { + "test": { "sum": { "field": "value" } } } } - ] - } + ], + "last_update_time": 1633392680364, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "SINGLE_ENTITY" + }, + "_primary_term": 1 } ``` -To set a category field for high cardinality: +To create high cardinality detector by specifying a category field: #### Request ```json POST _plugins/_anomaly_detection/detectors { - "name": "Host OK Rate Detector", - "description": "ok rate", - "time_field": "@timestamp", + "name": "test-hc-detector", + "description": "Test detector", + "time_field": "timestamp", "indices": [ - "host-cloudwatch" - ], - "category_field": [ - "host" + "server_log*" ], "feature_attributes": [ { - "feature_name": "latency_max", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "latency_max": { - "max": { - "field": "latency" + "test": { + "sum": { + "field": "value" } } } } ], - "window_delay": { - "period": { - "interval": 10, - "unit": "MINUTES" + "filter_query": { + "bool": { + "filter": [ + { + "range": { + "value": { + "gt": 1 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1 } }, "detection_interval": { "period": { "interval": 1, - "unit": "MINUTES" + "unit": "Minutes" } - } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "category_field": [ + "ip" + ] } ``` @@ -188,18 +223,32 @@ POST _plugins/_anomaly_detection/detectors ```json { - "_id": "4CIGoHUBTpMGN-4KzBQg", + "_id": "b0HRTXwBwf_U8gjUw43R", "_version": 1, - "_seq_no": 0, + "_seq_no": 6, "anomaly_detector": { - "name": "Host OK Rate Detector", - "description": "ok rate", - "time_field": "@timestamp", + "name": "test-hc-detector", + "description": "Test detector", + "time_field": "timestamp", "indices": [ - "server-metrics" + "server_log*" ], "filter_query": { - "match_all": { + "bool": { + "filter": [ + { + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } + } + } + ], + "adjust_pure_negative": true, "boost": 1 } }, @@ -211,35 +260,62 @@ POST _plugins/_anomaly_detection/detectors }, "window_delay": { "period": { - "interval": 10, - "unit": "MINUTES" + "interval": 1, + "unit": "Minutes" } }, - "shingle_size": 1, - "schema_version": 2, + "shingle_size": 8, + "schema_version": 0, "feature_attributes": [ { - "feature_id": "0Kld3HUBhpHMyt2e_UHn", - "feature_name": "latency_max", + "feature_id": "bkHRTXwBwf_U8gjUw43K", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "latency_max": { - "max": { - "field": "latency" + "test": { + "sum": { + "field": "value" } } } } ], - "last_update_time": 1604707601438, + "last_update_time": 1633393165265, "category_field": [ - "host" - ] + "ip" + ], + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "MULTI_ENTITY" }, "_primary_term": 1 } ``` +You can specify a maximum of two category fields: + +```json +"category_field": [ + "ip" +] +``` + +```json +"category_field": [ + "ip", "error_type" +] +``` + You can specify the following options. Options | Description | Type | Required @@ -272,65 +348,83 @@ GET _plugins/_anomaly_detection/detectors/ ```json { - "_id" : "m4ccEnIBTXsGi3mvMt9p", - "_version" : 1, - "_primary_term" : 1, - "_seq_no" : 3, - "anomaly_detector" : { - "name" : "test-detector", - "description" : "Test detector", - "time_field" : "timestamp", - "indices" : [ - "order*" + "_id": "VEHKTXwBwf_U8gjUXY2s", + "_version": 1, + "_primary_term": 1, + "_seq_no": 5, + "anomaly_detector": { + "name": "test-detector", + "description": "Test detector", + "time_field": "timestamp", + "indices": [ + "server_log*" ], - "filter_query" : { - "bool" : { - "filter" : [ + "filter_query": { + "bool": { + "filter": [ { - "exists" : { - "field" : "value", - "boost" : 1.0 + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } } } ], - "adjust_pure_negative" : true, - "boost" : 1.0 + "adjust_pure_negative": true, + "boost": 1 } }, - "detection_interval" : { - "period" : { - "interval" : 1, - "unit" : "Minutes" + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" } }, - "window_delay" : { - "period" : { - "interval" : 1, - "unit" : "Minutes" + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" } }, - "schema_version" : 0, - "feature_attributes" : [ + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ { - "feature_id" : "mYccEnIBTXsGi3mvMd8_", - "feature_name" : "total_order", - "feature_enabled" : true, - "aggregation_query" : { - "total_order" : { - "sum" : { - "field" : "value" + "feature_id": "U0HKTXwBwf_U8gjUXY2m", + "feature_name": "test", + "feature_enabled": true, + "aggregation_query": { + "test": { + "sum": { + "field": "value" } } } } ], - "last_update_time" : 1589441737319 + "last_update_time": 1633392680364, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "SINGLE_ENTITY" } } ``` - -Use `job=true` to get anomaly detection job information. +Use `task=true` to get real-time analysis task information. #### Request @@ -342,19 +436,33 @@ GET _plugins/_anomaly_detection/detectors/?job=true ```json { - "_id": "LJxGsXcBoDQA8W1Q--A1", + "_id": "VEHKTXwBwf_U8gjUXY2s", "_version": 1, "_primary_term": 1, - "_seq_no": 0, + "_seq_no": 5, "anomaly_detector": { - "name": "test2", - "description": "test", + "name": "test-detector", + "description": "Test detector", "time_field": "timestamp", "indices": [ - "server_log" + "server_log*" ], "filter_query": { - "match_all": { + "bool": { + "filter": [ + { + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } + } + } + ], + "adjust_pure_negative": true, "boost": 1 } }, @@ -374,11 +482,11 @@ GET _plugins/_anomaly_detection/detectors/?job=true "schema_version": 0, "feature_attributes": [ { - "feature_id": "K5xGsXcBoDQA8W1Q-uCF", - "feature_name": "F1", - "feature_enabled": "true", + "feature_id": "U0HKTXwBwf_U8gjUXY2m", + "feature_name": "test", + "feature_enabled": true, "aggregation_query": { - "f_1": { + "test": { "sum": { "field": "value" } @@ -386,14 +494,26 @@ GET _plugins/_anomaly_detection/detectors/?job=true } } ], - "last_update_time": 1613586955060, - "detector_type": "MULTI_ENTITY" + "last_update_time": 1633392680364, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "SINGLE_ENTITY" }, "anomaly_detector_job": { - "name": "LJxGsXcBoDQA8W1Q--A1", + "name": "VEHKTXwBwf_U8gjUXY2s", "schedule": { "interval": { - "start_time": 1613587220387, + "start_time": 1633393656357, "period": 1, "unit": "Minutes" } @@ -404,75 +524,28 @@ GET _plugins/_anomaly_detection/detectors/?job=true "unit": "Minutes" } }, - "enabled": "false", - "enabled_time": 1613587220387, - "last_update_time": 1613587289169, + "enabled": true, + "enabled_time": 1633393656357, + "last_update_time": 1633393656357, "lock_duration_seconds": 60, - "disabled_time": 1613587289169 - }, - "anomaly_detection_task": { - "task_id": "WZ5LsXcBoDQA8W1QmUa3", - "last_update_time": 1613587349022, - "error": "Task cancelled by user", - "state": "STOPPED", - "detector_id": "LJxGsXcBoDQA8W1Q--A1", - "task_progress": 0.26321793, - "init_progress": 1, - "current_piece": 1611030900000, - "execution_start_time": 1613587257783, - "execution_end_time": 1613587349022, - "is_latest": "true", - "task_type": "HISTORICAL", - "coordinating_node": "NSw5j-3YQeGkH8KESVKlzw", - "worker_node": "NSw5j-3YQeGkH8KESVKlzw", - "detector": { - "name": "test2", - "description": "test", - "time_field": "timestamp", - "indices": [ - "server_log" + "user": { + "name": "admin", + "backend_roles": [ + "admin" ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "K5xGsXcBoDQA8W1Q-uCF", - "feature_name": "F1", - "feature_enabled": "true", - "aggregation_query": { - "f_1": { - "sum": { - "field": "value" - } - } - } - } + "roles": [ + "own_index", + "all_access" ], - "last_update_time": 1613586955060, - "detector_type": "MULTI_ENTITY" + "custom_attribute_names": [], + "user_requested_tenant": "__user__" } } } ``` Use `task=true` to get historical analysis task information. +You can set both `job=true` and `task=true` to get information for both real-time and historical analysis tasks. #### Request @@ -484,25 +557,39 @@ GET _plugins/_anomaly_detection/detectors/?task=true ```json { - "_id": "BwzKQXcB89DLS7G9rg7Y", + "_id": "VEHKTXwBwf_U8gjUXY2s", "_version": 1, - "_primary_term": 2, - "_seq_no": 10, + "_primary_term": 1, + "_seq_no": 5, "anomaly_detector": { - "name": "test-ylwu1", - "description": "test", + "name": "test-detector", + "description": "Test detector", "time_field": "timestamp", "indices": [ - "ser*" + "server_log*" ], "filter_query": { - "match_all": { + "bool": { + "filter": [ + { + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } + } + } + ], + "adjust_pure_negative": true, "boost": 1 } }, "detection_interval": { "period": { - "interval": 10, + "interval": 1, "unit": "Minutes" } }, @@ -516,11 +603,11 @@ GET _plugins/_anomaly_detection/detectors/?task=true "schema_version": 0, "feature_attributes": [ { - "feature_id": "BgzKQXcB89DLS7G9rg7G", - "feature_name": "F1", + "feature_id": "U0HKTXwBwf_U8gjUXY2m", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "f_1": { + "test": { "sum": { "field": "value" } @@ -528,64 +615,63 @@ GET _plugins/_anomaly_detection/detectors/?task=true } } ], - "ui_metadata": { - "features": { - "F1": { - "aggregationBy": "sum", - "aggregationOf": "value", - "featureType": "simple_aggs" - } - } - }, - "last_update_time": 1611716538071, + "last_update_time": 1633392680364, "user": { "name": "admin", "backend_roles": [ "admin" ], "roles": [ - "all_access", - "own_index" + "own_index", + "all_access" ], "custom_attribute_names": [], "user_requested_tenant": "__user__" }, - "detector_type": "HISTORICAL_SINGLE_ENTITY", - "detection_date_range": { - "start_time": 1580094137997, - "end_time": 1611716537997 - } + "detector_type": "SINGLE_ENTITY" }, - "anomaly_detection_task": { - "task_id": "sgxaRXcB89DLS7G9RfIO", - "last_update_time": 1611776648699, + "realtime_detection_task": { + "task_id": "nkTZTXwBjd8s6RK4QlMq", + "last_update_time": 1633393776375, "started_by": "admin", - "state": "FINISHED", - "detector_id": "BwzKQXcB89DLS7G9rg7Y", - "task_progress": 1, + "error": "", + "state": "RUNNING", + "detector_id": "VEHKTXwBwf_U8gjUXY2s", + "task_progress": 0, "init_progress": 1, - "current_piece": 1611716400000, - "execution_start_time": 1611776279822, - "execution_end_time": 1611776648679, + "execution_start_time": 1633393656362, "is_latest": true, - "task_type": "HISTORICAL", - "coordinating_node": "gs213KqjS4q7H4Bmn_ZuLA", - "worker_node": "PgfR3JhbT7yJMx7bwQ6E3w", + "task_type": "REALTIME_SINGLE_ENTITY", + "coordinating_node": "SWD7ihu9TaaW1zKwFZNVNg", "detector": { - "name": "test-ylwu1", - "description": "test", + "name": "test-detector", + "description": "Test detector", "time_field": "timestamp", "indices": [ - "ser*" + "server_log*" ], "filter_query": { - "match_all": { + "bool": { + "filter": [ + { + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } + } + } + ], + "adjust_pure_negative": true, "boost": 1 } }, "detection_interval": { "period": { - "interval": 10, + "interval": 1, "unit": "Minutes" } }, @@ -599,11 +685,11 @@ GET _plugins/_anomaly_detection/detectors/?task=true "schema_version": 0, "feature_attributes": [ { - "feature_id": "BgzKQXcB89DLS7G9rg7G", - "feature_name": "F1", + "feature_id": "U0HKTXwBwf_U8gjUXY2m", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "f_1": { + "test": { "sum": { "field": "value" } @@ -611,33 +697,121 @@ GET _plugins/_anomaly_detection/detectors/?task=true } } ], - "ui_metadata": { - "features": { - "F1": { - "aggregationBy": "sum", - "aggregationOf": "value", - "featureType": "simple_aggs" - } - } - }, - "last_update_time": 1611716538071, + "last_update_time": 1633392680364, "user": { "name": "admin", "backend_roles": [ "admin" ], "roles": [ - "all_access", - "own_index" + "own_index", + "all_access" ], "custom_attribute_names": [], "user_requested_tenant": "__user__" }, - "detector_type": "HISTORICAL_SINGLE_ENTITY", - "detection_date_range": { - "start_time": 1580094137997, - "end_time": 1611716537997 - } + "detector_type": "SINGLE_ENTITY" + }, + "estimated_minutes_left": 0, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + } + }, + "historical_analysis_task": { + "task_id": "99DaTXwB6HknB84StRN1", + "last_update_time": 1633393797040, + "started_by": "admin", + "state": "RUNNING", + "detector_id": "VEHKTXwBwf_U8gjUXY2s", + "task_progress": 0.89285713, + "init_progress": 1, + "current_piece": 1633328940000, + "execution_start_time": 1633393751412, + "is_latest": true, + "task_type": "HISTORICAL_SINGLE_ENTITY", + "coordinating_node": "SWD7ihu9TaaW1zKwFZNVNg", + "worker_node": "2Z4q22BySEyzakYt_A0A2A", + "detector": { + "name": "test-detector", + "description": "Test detector", + "time_field": "timestamp", + "indices": [ + "server_log*" + ], + "filter_query": { + "bool": { + "filter": [ + { + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "U0HKTXwBwf_U8gjUXY2m", + "feature_name": "test", + "feature_enabled": true, + "aggregation_query": { + "test": { + "sum": { + "field": "value" + } + } + } + } + ], + "last_update_time": 1633392680364, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "SINGLE_ENTITY" + }, + "detection_date_range": { + "start_time": 1632788951329, + "end_time": 1633393751329 }, "user": { "name": "admin", @@ -645,8 +819,8 @@ GET _plugins/_anomaly_detection/detectors/?task=true "admin" ], "roles": [ - "all_access", - "own_index" + "own_index", + "all_access" ], "custom_attribute_names": [], "user_requested_tenant": "__user__" @@ -662,7 +836,10 @@ Introduced 1.0 {: .label .label-purple } Updates a detector with any changes, including the description or adding or removing of features. -To update a detector, you need to first stop the detector. +To update a detector, you need to first stop both real-time detection and historical analysis. + +You can't update a category field. +{: .note } #### Request @@ -670,17 +847,17 @@ To update a detector, you need to first stop the detector. PUT _plugins/_anomaly_detection/detectors/ { "name": "test-detector", - "description": "Test detector", + "description": "Test update detector", "time_field": "timestamp", "indices": [ - "order*" + "server_log*" ], "feature_attributes": [ { - "feature_name": "total_order", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "total_order": { + "test": { "sum": { "field": "value" } @@ -692,9 +869,10 @@ PUT _plugins/_anomaly_detection/detectors/ "bool": { "filter": [ { - "exists": { - "field": "value", - "boost": 1 + "range": { + "value": { + "gt": 1 + } } } ], @@ -704,14 +882,14 @@ PUT _plugins/_anomaly_detection/detectors/ }, "detection_interval": { "period": { - "interval": 10, - "unit": "MINUTES" + "interval": 1, + "unit": "Minutes" } }, "window_delay": { "period": { "interval": 1, - "unit": "MINUTES" + "unit": "Minutes" } } } @@ -722,59 +900,114 @@ PUT _plugins/_anomaly_detection/detectors/ ```json { - "_id" : "m4ccEnIBTXsGi3mvMt9p", - "_version" : 2, - "_seq_no" : 4, - "_primary_term" : 1, - "anomaly_detector" : { - "name" : "test-detector", - "description" : "Test detector", - "time_field" : "timestamp", - "indices" : [ - "order*" + "_id": "VEHKTXwBwf_U8gjUXY2s", + "_version": 2, + "_seq_no": 7, + "anomaly_detector": { + "name": "test-detector", + "description": "Test update detector", + "time_field": "timestamp", + "indices": [ + "server_log*" ], - "filter_query" : { - "bool" : { - "filter" : [ + "filter_query": { + "bool": { + "filter": [ { - "exists" : { - "field" : "value", - "boost" : 1.0 + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } } } ], - "adjust_pure_negative" : true, - "boost" : 1.0 + "adjust_pure_negative": true, + "boost": 1 } }, - "detection_interval" : { - "period" : { - "interval" : 10, - "unit" : "Minutes" + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" } }, - "window_delay" : { - "period" : { - "interval" : 1, - "unit" : "Minutes" + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" } }, - "schema_version" : 0, - "feature_attributes" : [ + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ { - "feature_id" : "xxokEnIBcpeWMD987A1X", - "feature_name" : "total_order", - "feature_enabled" : true, - "aggregation_query" : { - "total_order" : { - "sum" : { - "field" : "value" + "feature_id": "3kHiTXwBwf_U8gjUlY15", + "feature_name": "test", + "feature_enabled": true, + "aggregation_query": { + "test": { + "sum": { + "field": "value" } } } } - ] - } + ], + "last_update_time": 1633394267522, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "SINGLE_ENTITY" + }, + "_primary_term": 1 +} +``` + +--- + +## Delete detector +Introduced 1.0 +{: .label .label-purple } + +Deletes a detector based on the `detector_id`. +To delete a detector, you need to first stop both real-time detection and historical analysis. + +#### Request + +```json +DELETE _plugins/_anomaly_detection/detectors/ +``` + +#### Sample response + +```json +{ + "_index": ".opendistro-anomaly-detectors", + "_type": "_doc", + "_id": "70TxTXwBjd8s6RK4j1Pj", + "_version": 2, + "result": "deleted", + "forced_refresh": true, + "_shards": { + "total": 2, + "successful": 2, + "failed": 0 + }, + "_seq_no": 9, + "_primary_term": 1 } ``` @@ -786,20 +1019,114 @@ Introduced 1.0 Passes a date range to the anomaly detector to return any anomalies within that date range. +To preview a single-flow detector: + #### Request ```json POST _plugins/_anomaly_detection/detectors//_preview { - "period_start": 1612982516000, - "period_end": 1614278539000, + "period_start": 1633048868000, + "period_end": 1633394468000, "detector": { "name": "test-detector", - "description": "test server_log", + "description": "Test update detector", "time_field": "timestamp", "indices": [ - "server_log" + "server_log*" ], + "feature_attributes": [ + { + "feature_name": "test", + "feature_enabled": true, + "aggregation_query": { + "test": { + "sum": { + "field": "value" + } + } + } + } + ], + "filter_query": { + "bool": { + "filter": [ + { + "range": { + "value": { + "gt": 1 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + } + } +} +``` + +#### Sample response + +```json +{ + "anomaly_result": [ + { + "detector_id": null, + "data_start_time": 1633049280000, + "data_end_time": 1633049340000, + "schema_version": 0, + "feature_data": [ + { + "feature_id": "8EHmTXwBwf_U8gjU0Y0u", + "feature_name": "test", + "data": 0 + } + ], + "anomaly_grade": 0, + "confidence": 0 + }, + ... + ], + "anomaly_detector": { + "name": "test-detector", + "description": "Test update detector", + "time_field": "timestamp", + "indices": [ + "server_log*" + ], + "filter_query": { + "bool": { + "filter": [ + { + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1 + } + }, "detection_interval": { "period": { "interval": 1, @@ -812,18 +1139,85 @@ POST _plugins/_anomaly_detection/detectors//_preview "unit": "Minutes" } }, + "shingle_size": 8, + "schema_version": 0, "feature_attributes": [ { - "feature_name": "F1", + "feature_id": "8EHmTXwBwf_U8gjU0Y0u", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "f_1": { + "test": { "sum": { "field": "value" } } } } + ], + "detector_type": "SINGLE_ENTITY" + } +} +``` + +If you specify a category field, each result is associated with an entity: + +#### Request + +```json +POST _plugins/_anomaly_detection/detectors//_preview +{ + "period_start": 1633048868000, + "period_end": 1633394468000, + "detector": { + "name": "test-detector", + "description": "Test update detector", + "time_field": "timestamp", + "indices": [ + "server_log*" + ], + "feature_attributes": [ + { + "feature_name": "test", + "feature_enabled": true, + "aggregation_query": { + "test": { + "sum": { + "field": "value" + } + } + } + } + ], + "filter_query": { + "bool": { + "filter": [ + { + "range": { + "value": { + "gt": 1 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "category_field": [ + "error_type" ] } } @@ -834,37 +1228,48 @@ POST _plugins/_anomaly_detection/detectors//_preview ```json { "anomaly_result": [ - ... { - "detector_id": "m4ccEnIBTXsGi3mvMt9p", - "data_start_time": 1588843020000, - "data_end_time": 1588843620000, + "detector_id": null, + "data_start_time": 1633049280000, + "data_end_time": 1633049340000, + "schema_version": 0, "feature_data": [ { - "feature_id": "xxokEnIBcpeWMD987A1X", - "feature_name": "total_order", - "data": 489.9929131106 + "feature_id": "tkTpTXwBjd8s6RK4DlOZ", + "feature_name": "test", + "data": 0 } ], "anomaly_grade": 0, - "confidence": 0.99 - } + "confidence": 0, + "entity": [ + { + "name": "error_type", + "value": "error1" + } + ] + }, ... ], "anomaly_detector": { "name": "test-detector", - "description": "Test detector", + "description": "Test update detector", "time_field": "timestamp", "indices": [ - "order*" + "server_log*" ], "filter_query": { "bool": { "filter": [ { - "exists": { - "field": "value", - "boost": 1 + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } } } ], @@ -874,24 +1279,25 @@ POST _plugins/_anomaly_detection/detectors//_preview }, "detection_interval": { "period": { - "interval": 10, - "unit": "MINUTES" + "interval": 1, + "unit": "Minutes" } }, "window_delay": { "period": { "interval": 1, - "unit": "MINUTES" + "unit": "Minutes" } }, + "shingle_size": 8, "schema_version": 0, "feature_attributes": [ { - "feature_id": "xxokEnIBcpeWMD987A1X", - "feature_name": "total_order", + "feature_id": "tkTpTXwBjd8s6RK4DlOZ", + "feature_name": "test", "feature_enabled": true, "aggregation_query": { - "total_order": { + "test": { "sum": { "field": "value" } @@ -899,12 +1305,34 @@ POST _plugins/_anomaly_detection/detectors//_preview } } ], - "last_update_time": 1589442309241 + "category_field": [ + "error_type" + ], + "detector_type": "MULTI_ENTITY" } } ``` -If you specify a category field, each result is associated with an entity: +You can preview a detector with the detector ID: + +```json +POST _plugins/_anomaly_detection/detectors/_preview +{ + "detector_id": "VEHKTXwBwf_U8gjUXY2s", + "period_start": 1633048868000, + "period_end": 1633394468000 +} +``` + +Or: + +```json +POST _opendistro/_anomaly_detection/detectors/VEHKTXwBwf_U8gjUXY2s/_preview +{ + "period_start": 1633048868000, + "period_end": 1633394468000 +} +``` #### Sample response @@ -912,53 +1340,103 @@ If you specify a category field, each result is associated with an entity: { "anomaly_result": [ { - "detector_id": "4CIGoHUBTpMGN-4KzBQg", - "data_start_time": 1604277960000, - "data_end_time": 1604278020000, + "detector_id": "VEHKTXwBwf_U8gjUXY2s", + "data_start_time": 1633049280000, + "data_end_time": 1633049340000, "schema_version": 0, + "feature_data": [ + { + "feature_id": "3kHiTXwBwf_U8gjUlY15", + "feature_name": "test", + "data": 0 + } + ], "anomaly_grade": 0, - "confidence": 0.99 - } + "confidence": 0, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + } + }, + ... ], - "entity": [ - { - "name": "host", - "value": "i-00f28ec1eb8997686" - } - ] -}, -{ - "detector_id": "4CIGoHUBTpMGN-4KzBQg", - "data_start_time": 1604278020000, - "data_end_time": 1604278080000, - "schema_version": 0, - "feature_data": [ - { - "feature_id": "0Kld3HUBhpHMyt2e_UHn", - "feature_name": "latency_max", - "data": -17 - } - ], - "anomaly_grade": 0, - "confidence": 0.99, - "entity": [ - { - "name": "host", - "value": "i-00f28ec1eb8997686" - } - ] -} -... -``` - -Or, you can specify the detector ID: - -```json -POST _plugins/_anomaly_detection/detectors/_preview -{ - "detector_id": "sYkUvHcBiZv51f-Lv8QN", - "period_start": 1612982516000, - "period_end": 1614278539000 + "anomaly_detector": { + "name": "test-detector", + "description": "Test update detector", + "time_field": "timestamp", + "indices": [ + "server_log*" + ], + "filter_query": { + "bool": { + "filter": [ + { + "range": { + "value": { + "from": 1, + "to": null, + "include_lower": false, + "include_upper": true, + "boost": 1 + } + } + } + ], + "adjust_pure_negative": true, + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "3kHiTXwBwf_U8gjUlY15", + "feature_name": "test", + "feature_enabled": true, + "aggregation_query": { + "test": { + "sum": { + "field": "value" + } + } + } + } + ], + "last_update_time": 1633394267522, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "SINGLE_ENTITY" + } } ``` @@ -970,6 +1448,7 @@ Introduced 1.0 Starts a real-time or historical anomaly detector job. +To start a real-time detector job: #### Request @@ -981,23 +1460,38 @@ POST _plugins/_anomaly_detection/detectors//_start ```json { - "_id" : "m4ccEnIBTXsGi3mvMt9p", - "_version" : 1, - "_seq_no" : 6, - "_primary_term" : 1 + "_id": "VEHKTXwBwf_U8gjUXY2s", + "_version": 3, + "_seq_no": 6, + "_primary_term": 1 } ``` +The `_id` represents the real-time job ID, which is the same as the detector ID. + To start historical analysis: ```json POST _plugins/_anomaly_detection/detectors//_start { - "start_time": 1503168590000, - "end_time": 1617301324000 + "start_time": 1633048868000, + "end_time": 1633394468000 } ``` +#### Sample response + +```json +{ + "_id": "f9DsTXwB6HknB84SoRTY", + "_version": 1, + "_seq_no": 958, + "_primary_term": 1 +} +``` + +The `_id` represents the historical batch task ID, which is a random universally unique identifier (UUID). + --- ## Stop detector job @@ -1006,6 +1500,8 @@ Introduced 1.0 Stops a real-time or historical anomaly detector job. +To stop a real-time detector job: + #### Request ```json @@ -1015,7 +1511,12 @@ POST _plugins/_anomaly_detection/detectors//_stop #### Sample response ```json -Stopped detector: m4ccEnIBTXsGi3mvMt9p +{ + "_id": "VEHKTXwBwf_U8gjUXY2s", + "_version": 0, + "_seq_no": 0, + "_primary_term": 0 +} ``` To stop historical analysis: @@ -1024,30 +1525,37 @@ To stop historical analysis: POST _plugins/_anomaly_detection/detectors//_stop?historical=true ``` +#### Sample response + +```json +{ + "_id": "f9DsTXwB6HknB84SoRTY", + "_version": 0, + "_seq_no": 0, + "_primary_term": 0 +} +``` --- -## Search detector result +## Search detector Introduced 1.0 {: .label .label-purple } -Returns all results for a search query. +Returns all anomaly detectors for a search query. + +To search detectors using the `server_log*` index: #### Request ```json -GET _plugins/_anomaly_detection/detectors/results/_search -POST _plugins/_anomaly_detection/detectors/results/_search +GET _plugins/_anomaly_detection/detectors/_search +POST _plugins/_anomaly_detection/detectors/_search { "query": { - "bool": { - "must": { - "range": { - "anomaly_score": { - "gte": 0.6, - "lte": 1 - } - } + "wildcard": { + "indices": { + "value": "server_log*" } } } @@ -1058,138 +1566,7 @@ POST _plugins/_anomaly_detection/detectors/results/_search ```json { - "took": 9, - "timed_out": false, - "_shards": { - "total": 25, - "successful": 25, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": { - "value": 2, - "relation": "eq" - }, - "max_score": 1, - "hits": [ - { - "_index": ".opensearch-anomaly-results-history-2020.04.30-1", - "_type": "_doc", - "_id": "_KBrzXEBbpoKkFM5mStm", - "_version": 1, - "_seq_no": 58, - "_primary_term": 1, - "_score": 1, - "_source": { - "detector_id": "2KDozHEBbpoKkFM58yr6", - "anomaly_score": 0.8995068350366767, - "execution_start_time": 1588289313114, - "data_end_time": 1588289313114, - "confidence": 0.84214852704501, - "data_start_time": 1588289253114, - "feature_data": [ - { - "feature_id": "X0fpzHEB5NGZmIRkXKcy", - "feature_name": "total_error", - "data": 20 - } - ], - "execution_end_time": 1588289313126, - "anomaly_grade": 0 - } - }, - { - "_index": ".opensearch-anomaly-results-history-2020.04.30-1", - "_type": "_doc", - "_id": "EqB1zXEBbpoKkFM5qyyE", - "_version": 1, - "_seq_no": 61, - "_primary_term": 1, - "_score": 1, - "_source": { - "detector_id": "2KDozHEBbpoKkFM58yr6", - "anomaly_score": 0.7086834513354907, - "execution_start_time": 1588289973113, - "data_end_time": 1588289973113, - "confidence": 0.42162017029510446, - "data_start_time": 1588289913113, - "feature_data": [ - { - "feature_id": "X0fpzHEB5NGZmIRkXKcy", - "feature_name": "memory_usage", - "data": 20.0347333108 - } - ], - "execution_end_time": 1588289973124, - "anomaly_grade": 0 - } - } - ] - } -} -``` - -In high cardinality detectors, the result contains entity information. - -To see an ordered set of anomaly records for an entity with an anomaly within a certain time range for a specific feature value: - -#### Request - -```json -POST _plugins/_anomaly_detection/detectors/results/_search -{ - "query": { - "bool": { - "filter": [ - { - "term": { - "detector_id": "4CIGoHUBTpMGN-4KzBQg" - } - }, - { - "range": { - "anomaly_grade": { - "gt": 0 - } - } - }, - { - "nested": { - "path": "entity", - "query": { - "bool": { - "must": [ - { - "term": { - "entity.value": "i-00f28ec1eb8997685" - } - } - ] - } - } - } - } - ] - } - }, - "size": 8, - "sort": [ - { - "execution_end_time": { - "order": "desc" - } - } - ], - "track_total_hits": true -} -``` - -#### Sample response - -```json -{ - "took": 443, + "took": 2, "timed_out": false, "_shards": { "total": 1, @@ -1199,136 +1576,119 @@ POST _plugins/_anomaly_detection/detectors/results/_search }, "hits": { "total": { - "value": 7, + "value": 4, "relation": "eq" }, - "max_score": null, + "max_score": 1, "hits": [ { - "_index": ".opensearch-anomaly-results-history-2020.11.07-1", + "_index": ".opendistro-anomaly-detectors", "_type": "_doc", - "_id": "BiItoHUBTpMGN-4KARY5", + "_id": "Zi5zTXwBwf_U8gjUTfJG", "_version": 1, - "_seq_no": 206, + "_seq_no": 1, "_primary_term": 1, - "_score": null, + "_score": 1, "_source": { - "detector_id": "4CIGoHUBTpMGN-4KzBQg", - "schema_version": 2, - "anomaly_score": 2.462550517055763, - "execution_start_time": 1604710105400, - "data_end_time": 1604710094516, - "confidence": 0.8246254862573076, - "data_start_time": 1604710034516, - "feature_data": [ + "name": "test", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" + ], + "filter_query": { + "match_all": { + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 5, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ { - "feature_id": "0Kld3HUBhpHMyt2e_UHn", - "feature_name": "latency_max", - "data": 3526 + "feature_id": "ZS5zTXwBwf_U8gjUTfIn", + "feature_name": "test_feature", + "feature_enabled": true, + "aggregation_query": { + "test_feature": { + "sum": { + "field": "value" + } + } + } } ], - "execution_end_time": 1604710105401, - "anomaly_grade": 0.08045977011494891, - "entity": [ - { - "name": "host", - "value": "i-00f28ec1eb8997685" - } - ] - }, - "sort": [ - 1604710105401 - ] + "last_update_time": 1633386974533, + "category_field": [ + "error_type" + ], + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "MULTI_ENTITY" + } }, - { - "_index": ".opensearch-anomaly-results-history-2020.11.07-1", - "_type": "_doc", - "_id": "wiImoHUBTpMGN-4KlhXs", - "_version": 1, - "_seq_no": 156, - "_primary_term": 1, - "_score": null, - "_source": { - "detector_id": "4CIGoHUBTpMGN-4KzBQg", - "schema_version": 2, - "anomaly_score": 4.892453213261217, - "execution_start_time": 1604709684971, - "data_end_time": 1604709674522, - "confidence": 0.8313735633713821, - "data_start_time": 1604709614522, - "feature_data": [ - { - "feature_id": "0Kld3HUBhpHMyt2e_UHn", - "feature_name": "latency_max", - "data": 5709 - } - ], - "execution_end_time": 1604709684971, - "anomaly_grade": 0.06542056074767538, - "entity": [ - { - "name": "host", - "value": "i-00f28ec1eb8997685" - } - ] - }, - "sort": [ - 1604709684971 - ] - }, - { - "_index": ".opensearch-anomaly-results-history-2020.11.07-1", - "_type": "_doc", - "_id": "ZiIcoHUBTpMGN-4KhhVA", - "_version": 1, - "_seq_no": 79, - "_primary_term": 1, - "_score": null, - "_source": { - "detector_id": "4CIGoHUBTpMGN-4KzBQg", - "schema_version": 2, - "anomaly_score": 3.187717536855158, - "execution_start_time": 1604709025343, - "data_end_time": 1604709014520, - "confidence": 0.8301116064308817, - "data_start_time": 1604708954520, - "feature_data": [ - { - "feature_id": "0Kld3HUBhpHMyt2e_UHn", - "feature_name": "latency_max", - "data": 441 - } - ], - "execution_end_time": 1604709025344, - "anomaly_grade": 0.040767386091133916, - "entity": [ - { - "name": "host", - "value": "i-00f28ec1eb8997685" - } - ] - }, - "sort": [ - 1604709025344 - ] - } + ... ] } } ``` -You can query the anomaly results for historical analysis with the `task_id`: +--- + +## Search detector tasks +Introduced 1.1 +{: .label .label-purple } + +Searches detector tasks. + +To search for the latest detector task for a high cardinality detector: #### Request ```json -GET _plugins/_anomaly_detection/detectors/results/_search +GET _plugins/_anomaly_detection/detectors/tasks/_search +POST _plugins/_anomaly_detection/detectors/tasks/_search { "query": { - "term": { - "task_id": { - "value": "NnlV9HUBQxqfQ7vBJNzy" - } + "bool": { + "filter": [ + { + "term": { + "detector_id": "Zi5zTXwBwf_U8gjUTfJG" + } + }, + { + "term": { + "task_type": "HISTORICAL_HC_DETECTOR" + } + }, + { + "term": { + "is_latest": "true" + } + } + ] } } } @@ -1351,211 +1711,41 @@ GET _plugins/_anomaly_detection/detectors/results/_search "value": 1, "relation": "eq" }, - "max_score": 2.1366, + "max_score": 0, "hits": [ { - "_index": ".opensearch-anomaly-detection-state", + "_index": ".opendistro-anomaly-detection-state", "_type": "_doc", - "_id": "CoM8WncBtt2qvI-LZO7_", - "_version": 8, - "_seq_no": 1351, - "_primary_term": 3, - "_score": 2.1366, - "_source": { - "detector_id": "dZc8WncBgO2zoQoFWVBA", - "worker_node": "dk6-HuKQRMKm2fi8TSDHsg", - "task_progress": 0.09486946, - "last_update_time": 1612126667008, - "execution_start_time": 1612126643455, - "state": "RUNNING", - "coordinating_node": "gs213KqjS4q7H4Bmn_ZuLA", - "current_piece": 1583503800000, - "task_type": "HISTORICAL", - "started_by": "admin", - "init_progress": 1, - "is_latest": true, - "detector": { - "description": "test", - "ui_metadata": { - "features": { - "F1": { - "aggregationBy": "sum", - "aggregationOf": "value", - "featureType": "simple_aggs" - } - } - }, - "detection_date_range": { - "start_time": 1580504240308, - "end_time": 1612126640308 - }, - "feature_attributes": [ - { - "feature_id": "dJc8WncBgO2zoQoFWVAt", - "feature_enabled": true, - "feature_name": "F1", - "aggregation_query": { - "f_1": { - "sum": { - "field": "value" - } - } - } - } - ], - "schema_version": 0, - "time_field": "timestamp", - "last_update_time": 1612126640448, - "indices": [ - "server_log" - ], - "window_delay": { - "period": { - "unit": "Minutes", - "interval": 1 - } - }, - "detection_interval": { - "period": { - "unit": "Minutes", - "interval": 10 - } - }, - "name": "test-historical-detector", - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "shingle_size": 8, - "user": { - "backend_roles": [ - "admin" - ], - "custom_attribute_names": [], - "roles": [ - "all_access", - "own_index" - ], - "name": "admin", - "user_requested_tenant": "__user__" - }, - "detector_type": "HISTORICAL_SINGLE_ENTITY" - }, - "user": { - "backend_roles": [ - "admin" - ], - "custom_attribute_names": [], - "roles": [ - "all_access", - "own_index" - ], - "name": "admin", - "user_requested_tenant": "__user__" - } - } - } - ] - } -} -``` - ---- - -## Search detector tasks -Introduced 1.1 -{: .label .label-purple } - -Searches detector tasks. - -#### Request - -```json -POST _plugins/_anomaly_detection/detectors/tasks/_search -{ - "query": { - "bool": { - "filter": [ - { - "term": { - "detector_id": { - "value": "_6WPu3cBBnauGn7oxUAv" - } - } - }, - { - "term": { - "task_type": { - "value": "HISTORICAL_HC_DETECTOR" - } - } - } - ] - } - }, - "sort": [ - { - "execution_start_time": { - "order": "desc" - } - } - ] -} -``` - - -#### Sample response - -```json -{ - "took": 5, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": { - "value": 1, - "relation": "eq" - }, - "max_score": null, - "hits": [ - { - "_index": ".opensearch-anomaly-detection-state", - "_type": "_doc", - "_id": "TM3tOHwBCi2h__AOXlyQ", - "_version": 3, - "_seq_no": 14, + "_id": "fm-RTXwBYwCbWecgB753", + "_version": 34, + "_seq_no": 928, "_primary_term": 1, - "_score": null, + "_score": 0, "_source": { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", + "detector_id": "Zi5zTXwBwf_U8gjUTfJG", "error": "", "detection_date_range": { - "start_time": 1632437852100, - "end_time": 1633042652100 + "start_time": 1630794960000, + "end_time": 1633386960000 }, - "task_progress": 0.5, - "last_update_time": 1633042667358, - "execution_start_time": 1633042652810, - "state": "RUNNING", - "coordinating_node": "2hEGbUw6ShaiKe05n_xLdA", + "task_progress": 1, + "last_update_time": 1633389090738, + "execution_start_time": 1633388922742, + "state": "FINISHED", + "coordinating_node": "2Z4q22BySEyzakYt_A0A2A", "task_type": "HISTORICAL_HC_DETECTOR", + "execution_end_time": 1633389090738, "started_by": "admin", "init_progress": 0, "is_latest": true, "detector": { "category_field": [ - "type" + "error_type" ], "description": "test", "ui_metadata": { "features": { - "test-feature": { + "test_feature": { "aggregationBy": "sum", "aggregationOf": "value", "featureType": "simple_aggs" @@ -1565,9 +1755,9 @@ POST _plugins/_anomaly_detection/detectors/tasks/_search }, "feature_attributes": [ { - "feature_id": "7VDtOHwBD5tpxlbyWqPs", + "feature_id": "ZS5zTXwBwf_U8gjUTfIn", "feature_enabled": true, - "feature_name": "test-feature", + "feature_name": "test_feature", "aggregation_query": { "test_feature": { "sum": { @@ -1579,7 +1769,7 @@ POST _plugins/_anomaly_detection/detectors/tasks/_search ], "schema_version": 0, "time_field": "timestamp", - "last_update_time": 1633042652012, + "last_update_time": 1633386974533, "indices": [ "server_log" ], @@ -1595,7 +1785,7 @@ POST _plugins/_anomaly_detection/detectors/tasks/_search "interval": 5 } }, - "name": "test-detector", + "name": "testhc", "filter_query": { "match_all": { "boost": 1 @@ -1612,7 +1802,7 @@ POST _plugins/_anomaly_detection/detectors/tasks/_search "all_access" ], "name": "admin", - "user_requested_tenant": null + "user_requested_tenant": "__user__" }, "detector_type": "MULTI_ENTITY" }, @@ -1628,50 +1818,1060 @@ POST _plugins/_anomaly_detection/detectors/tasks/_search "name": "admin", "user_requested_tenant": "__user__" } - }, - "sort": [ - 1633042652810 - ] + } } ] } } ``` - ---- - -## Delete detector -Introduced 1.0 -{: .label .label-purple } - -Deletes a detector based on the `detector_id`. -To delete a detector, you need to first stop the detector. +To search for the latest entity-level task for a high cardinality detector: #### Request ```json -DELETE _plugins/_anomaly_detection/detectors/ +GET _plugins/_anomaly_detection/detectors/tasks/_search +POST _plugins/_anomaly_detection/detectors/tasks/_search +{ + "query": { + "bool": { + "filter": [ + { + "term": { + "detector_id": "Zi5zTXwBwf_U8gjUTfJG" + } + }, + { + "term": { + "task_type": "HISTORICAL_HC_ENTITY" + } + }, + { + "term": { + "is_latest": "true" + } + } + ] + } + }, + "sort": [ + { + "execution_start_time": { + "order": "desc" + } + } + ], + "size": 100 +} ``` +To search for all entity-level batch task stats: + +#### Request + +```json +GET _plugins/_anomaly_detection/detectors/tasks/_search +POST _plugins/_anomaly_detection/detectors/tasks/_search +{ + "size": 0, + "query": { + "bool": { + "filter": [ + { + "term": { + "detector_id": { + "value": "Zi5zTXwBwf_U8gjUTfJG", + "boost": 1 + } + } + }, + { + "term": { + "parent_task_id": { + "value": "fm-RTXwBYwCbWecgB753", + "boost": 1 + } + } + }, + { + "terms": { + "task_type": [ + "HISTORICAL_HC_ENTITY" + ], + "boost": 1 + } + } + ] + } + }, + "aggs": { + "test": { + "terms": { + "field": "state", + "size": 100 + } + } + } +} +``` #### Sample response ```json { - "_index": ".opensearch-anomaly-detectors", - "_type": "_doc", - "_id": "m4ccEnIBTXsGi3mvMt9p", - "_version": 2, - "result": "deleted", - "forced_refresh": true, + "took": 2, + "timed_out": false, "_shards": { - "total": 2, - "successful": 2, + "total": 1, + "successful": 1, + "skipped": 0, "failed": 0 }, - "_seq_no": 6, - "_primary_term": 1 + "hits": { + "total": { + "value": 32, + "relation": "eq" + }, + "max_score": null, + "hits": [] + }, + "aggregations": { + "test": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "FINISHED", + "doc_count": 32 + } + ] + } + } +} +``` + +--- + +## Search detector result +Introduced 1.0 +{: .label .label-purple } + +Returns all results for a search query. + +To search anomaly results for `grade` greater than 0 for real-time analysis: + +#### Request + +```json +GET _plugins/_anomaly_detection/detectors/results/_search +POST _plugins/_anomaly_detection/detectors/results/_search +{ + "query": { + "bool": { + "filter": [ + { + "term": { + "detector_id": "Zi5zTXwBwf_U8gjUTfJG" + } + }, + { + "range": { + "anomaly_grade": { + "gt": 0 + } + } + } + ], + "must_not": [ + { + "exists": { + "field": "task_id" + } + } + ] + } + } +} +``` + +Because real-time analysis doesn't have a batch task, the task ID in the anomaly result is null. + +#### Sample response + +```json +{ + "took": 4, + "timed_out": false, + "_shards": { + "total": 3, + "successful": 3, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 90, + "relation": "eq" + }, + "max_score": 0, + "hits": [ + { + "_index": ".opendistro-anomaly-results-history-2021.10.04-1", + "_type": "_doc", + "_id": "686KTXwB6HknB84SMr6G", + "_version": 1, + "_seq_no": 103622, + "_primary_term": 1, + "_score": 0, + "_source": { + "detector_id": "Zi5zTXwBwf_U8gjUTfJG", + "confidence": 0.918886275269358, + "model_id": "Zi5zTXwBwf_U8gjUTfJG_entity_error16", + "schema_version": 4, + "anomaly_score": 1.1093755891885446, + "execution_start_time": 1633388475001, + "data_end_time": 1633388414989, + "data_start_time": 1633388114989, + "feature_data": [ + { + "feature_id": "ZS5zTXwBwf_U8gjUTfIn", + "feature_name": "test_feature", + "data": 0.532 + } + ], + "execution_end_time": 1633388475014, + "user": { + "backend_roles": [ + "admin" + ], + "custom_attribute_names": [], + "roles": [ + "own_index", + "all_access" + ], + "name": "admin", + "user_requested_tenant": "__user__" + }, + "anomaly_grade": 0.031023547546561225, + "entity": [ + { + "name": "error_type", + "value": "error16" + } + ] + } + }, + ... + ] + } +} +``` + +You can run historical analysis as many times as you like. So, multiple tasks might exist for the same detector. + +You can search for the latest historical batch task first and then search the historical batch task results. + +To search anomaly results for `grade` greater than 0 for historical analysis with the `task_id`: + +#### Request + +```json +GET _plugins/_anomaly_detection/detectors/results/_search +POST _plugins/_anomaly_detection/detectors/results/_search +{ + "query": { + "bool": { + "filter": [ + { + "term": { + "detector_id": "Zi5zTXwBwf_U8gjUTfJG" + } + }, + { + "range": { + "anomaly_grade": { + "gt": 0 + } + } + }, + { + "term": { + "task_id": "fm-RTXwBYwCbWecgB753" + } + } + ] + } + } +} +``` + +#### Sample response + +```json +{ + "took": 915, + "timed_out": false, + "_shards": { + "total": 3, + "successful": 3, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 4115, + "relation": "eq" + }, + "max_score": 0, + "hits": [ + { + "_index": ".opendistro-anomaly-results-history-2021.10.04-1", + "_type": "_doc", + "_id": "VRyRTXwBDx7vzPBV8jYC", + "_version": 1, + "_seq_no": 149657, + "_primary_term": 1, + "_score": 0, + "_source": { + "detector_id": "Zi5zTXwBwf_U8gjUTfJG", + "confidence": 0.9642989263957601, + "task_id": "fm-RTXwBYwCbWecgB753", + "model_id": "Zi5zTXwBwf_U8gjUTfJG_entity_error24", + "schema_version": 4, + "anomaly_score": 1.2260712437521946, + "execution_start_time": 1633388982692, + "data_end_time": 1631721300000, + "data_start_time": 1631721000000, + "feature_data": [ + { + "feature_id": "ZS5zTXwBwf_U8gjUTfIn", + "feature_name": "test_feature", + "data": 10 + } + ], + "execution_end_time": 1633388982709, + "user": { + "backend_roles": [ + "admin" + ], + "custom_attribute_names": [], + "roles": [ + "own_index", + "all_access" + ], + "name": "admin", + "user_requested_tenant": "__user__" + }, + "anomaly_grade": 0.14249628345655782, + "entity": [ + { + "name": "error_type", + "value": "error1" + } + ] + } + }, + ... + ] + } +} +``` + +--- + +## Get detector stats +Introduced 1.0 +{: .label .label-purple } + +Provides information about how the plugin is performing. + +To get all stats: + +#### Request + +```json +GET _plugins/_anomaly_detection/stats +``` + +#### Sample response + +```json +{ + "anomaly_detectors_index_status": "green", + "anomaly_detection_state_status": "green", + "single_entity_detector_count": 2, + "detector_count": 5, + "multi_entity_detector_count": 3, + "anomaly_detection_job_index_status": "green", + "models_checkpoint_index_status": "green", + "anomaly_results_index_status": "green", + "nodes": { + "2Z4q22BySEyzakYt_A0A2A": { + "ad_execute_request_count": 95, + "models": [ + { + "detector_id": "WTBnTXwBjd8s6RK4b1Sz", + "model_type": "rcf", + "last_used_time": 1633398197185, + "model_id": "WTBnTXwBjd8s6RK4b1Sz_model_rcf_0", + "last_checkpoint_time": 1633396573679 + }, + ... + ], + "ad_canceled_batch_task_count": 0, + "ad_hc_execute_request_count": 75, + "ad_hc_execute_failure_count": 0, + "model_count": 28, + "ad_execute_failure_count": 1, + "ad_batch_task_failure_count": 0, + "ad_total_batch_task_execution_count": 27, + "ad_executing_batch_task_count": 3 + }, + "SWD7ihu9TaaW1zKwFZNVNg": { + "ad_execute_request_count": 12, + "models": [ + { + "detector_id": "Zi5zTXwBwf_U8gjUTfJG", + "model_type": "entity", + "last_used_time": 1633398375008, + "model_id": "Zi5zTXwBwf_U8gjUTfJG_entity_error13", + "last_checkpoint_time": 1633392973682, + "entity": [ + { + "name": "error_type", + "value": "error13" + } + ] + }, + ... + ], + "ad_canceled_batch_task_count": 1, + "ad_hc_execute_request_count": 0, + "ad_hc_execute_failure_count": 0, + "model_count": 15, + "ad_execute_failure_count": 2, + "ad_batch_task_failure_count": 0, + "ad_total_batch_task_execution_count": 27, + "ad_executing_batch_task_count": 4 + }, + "TQDUXEzyTJyV0H6_T4hYUw": { + "ad_execute_request_count": 0, + "models": [ + { + "detector_id": "Zi5zTXwBwf_U8gjUTfJG", + "model_type": "entity", + "last_used_time": 1633398375004, + "model_id": "Zi5zTXwBwf_U8gjUTfJG_entity_error24", + "last_checkpoint_time": 1633388177359, + "entity": [ + { + "name": "error_type", + "value": "error24" + } + ] + }, + ... + ], + "ad_canceled_batch_task_count": 0, + "ad_hc_execute_request_count": 0, + "ad_hc_execute_failure_count": 0, + "model_count": 22, + "ad_execute_failure_count": 0, + "ad_batch_task_failure_count": 0, + "ad_total_batch_task_execution_count": 28, + "ad_executing_batch_task_count": 3 + } + } +} +``` + +The `model_count` parameter shows the total number of models running on each node’s memory. +For historical analysis, you see the values for the following fields: + +- `ad_total_batch_task_execution_count` +- `ad_executing_batch_task_count` +- `ad_canceled_batch_task_count` +- `ad_batch_task_failure_count` + +If haven't run any historical analysis, these values show up as 0. + +To get all stats for a specific node: + +#### Request + +```json +GET _plugins/_anomaly_detection//stats +``` + +To get specific stats for a node: + +#### Request + +```json +GET _plugins/_anomaly_detection//stats/ +GET _plugins/_anomaly_detection//stats/ad_execute_request_count +``` + +#### Sample response + +```json +{ + "nodes": { + "SWD7ihu9TaaW1zKwFZNVNg": { + "ad_execute_request_count": 12 + } + } +} +``` + +To get a specific type of stats: + +#### Request + +```json +GET _plugins/_anomaly_detection/stats/ +GET _plugins/_anomaly_detection/stats/ad_executing_batch_task_count +``` + +#### Sample response + +```json +{ + "nodes": { + "2Z4q22BySEyzakYt_A0A2A": { + "ad_executing_batch_task_count": 3 + }, + "SWD7ihu9TaaW1zKwFZNVNg": { + "ad_executing_batch_task_count": 3 + }, + "TQDUXEzyTJyV0H6_T4hYUw": { + "ad_executing_batch_task_count": 4 + } + } +} +``` + +--- + +## Profile detector +Introduced 1.0 +{: .label .label-purple } + +Returns information related to the current state of the detector and memory usage, including current errors and shingle size, to help troubleshoot the detector. + +This command helps locate logs by identifying the nodes that run the anomaly detector job for each detector. + +It also helps track the initialization percentage, the required shingles, and the estimated time left. + +#### Request + +```json +GET _plugins/_anomaly_detection/detectors//_profile/ +GET _plugins/_anomaly_detection/detectors//_profile?_all=true +GET _plugins/_anomaly_detection/detectors//_profile/ +GET _plugins/_anomaly_detection/detectors//_profile/, +``` + +#### Sample Responses + +```json +GET _plugins/_anomaly_detection/detectors//_profile + +{ + "state": "DISABLED", + "error": "Stopped detector: AD models memory usage exceeds our limit." +} + +GET _plugins/_anomaly_detection/detectors//_profile?_all=true&pretty + +{ + "state": "RUNNING", + "error": "", + "models": [ + { + "model_id": "3Dh6TXwBwf_U8gjURE0F_entity_KSLSh0Wv05RQXiBAQHTEZg", + "entity": [ + { + "name": "ip", + "value": "192.168.1.1" + }, + { + "name": "error_type", + "value": "error8" + } + ], + "model_size_in_bytes": 403491, + "node_id": "2Z4q22BySEyzakYt_A0A2A" + }, + ... + ], + "total_size_in_bytes": 12911712, + "init_progress": { + "percentage": "100%" + }, + "total_entities": 33, + "active_entities": 32, + "ad_task": { + "ad_task": { + "task_id": "D3I5TnwBYwCbWecg7lN9", + "last_update_time": 1633399993685, + "started_by": "admin", + "state": "RUNNING", + "detector_id": "3Dh6TXwBwf_U8gjURE0F", + "task_progress": 0, + "init_progress": 0, + "execution_start_time": 1633399991933, + "is_latest": true, + "task_type": "HISTORICAL_HC_DETECTOR", + "coordinating_node": "2Z4q22BySEyzakYt_A0A2A", + "detector": { + "name": "testhc-mc", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" + ], + "filter_query": { + "match_all": { + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 5, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "2zh6TXwBwf_U8gjUQ039", + "feature_name": "test", + "feature_enabled": true, + "aggregation_query": { + "test": { + "sum": { + "field": "value" + } + } + } + } + ], + "ui_metadata": { + "features": { + "test": { + "aggregationBy": "sum", + "aggregationOf": "value", + "featureType": "simple_aggs" + } + }, + "filters": [] + }, + "last_update_time": 1633387430916, + "category_field": [ + "ip", + "error_type" + ], + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "MULTI_ENTITY" + }, + "detection_date_range": { + "start_time": 1632793800000, + "end_time": 1633398600000 + }, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + } + }, + "node_id": "2Z4q22BySEyzakYt_A0A2A", + "task_id": "D3I5TnwBYwCbWecg7lN9", + "task_type": "HISTORICAL_HC_DETECTOR", + "detector_task_slots": 10, + "total_entities_count": 32, + "pending_entities_count": 22, + "running_entities_count": 10, + "running_entities": [ """[{"name":"ip","value":"192.168.1.1"},{"name":"error_type","value":"error9"}]""", + ...], + "entity_task_profiles": [ + { + "shingle_size": 8, + "rcf_total_updates": 1994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "2Z4q22BySEyzakYt_A0A2A", + "entity": [ + { + "name": "ip", + "value": "192.168.1.1" + }, + { + "name": "error_type", + "value": "error7" + } + ], + "task_id": "E3I5TnwBYwCbWecg9FMm", + "task_type": "HISTORICAL_HC_ENTITY" + }, + ... + ] + }, + "model_count": 32 +} + +GET _plugins/_anomaly_detection/detectors//_profile/total_size_in_bytes + +{ + "total_size_in_bytes": 13369344 +} +``` + +You can see the `ad_task` field only for historical analysis. + +The `model_count` parameter shows the total number of models that a detector runs on each node’s memory. This is useful if you have several models running on your cluster and want to know the count. + +If you configured the category field, you can see the number of unique values in the field and all active entities with models running in memory. + +You can use this data to estimate how much memory is required for anomaly detection so you can decide how to size your cluster. For example, if a detector has one million entities and only 10 of them are active in memory, you need to scale your cluster up or out. + +For a single-entity detector: + +#### Sample response + +```json +{ + "state": "INIT", + "total_size_in_bytes": 0, + "init_progress": { + "percentage": "0%", + "needed_shingles": 128 + }, + "ad_task": { + "ad_task": { + "task_id": "cfUNOXwBFLNqSEcxAlde", + "last_update_time": 1633044731640, + "started_by": "admin", + "state": "RUNNING", + "detector_id": "qL4NOXwB__6eNorTAKtJ", + "task_progress": 0.49603173, + "init_progress": 1, + "current_piece": 1632739800000, + "execution_start_time": 1633044726365, + "is_latest": true, + "task_type": "HISTORICAL_SINGLE_ENTITY", + "coordinating_node": "bCtWtxWPThq0BIn5P5I4Xw", + "worker_node": "dIyavWhmSYWGz65b4u-lpQ", + "detector": { + "name": "detector1", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" + ], + "filter_query": { + "match_all": { + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 5, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "p74NOXwB__6eNorTAKss", + "feature_name": "test-feature", + "feature_enabled": true, + "aggregation_query": { + "test_feature": { + "sum": { + "field": "value" + } + } + } + } + ], + "ui_metadata": { + "features": { + "test-feature": { + "aggregationBy": "sum", + "aggregationOf": "value", + "featureType": "simple_aggs" + } + }, + "filters": [] + }, + "last_update_time": 1633044725832, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "SINGLE_ENTITY" + }, + "detection_date_range": { + "start_time": 1632439925885, + "end_time": 1633044725885 + }, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + } + }, + "shingle_size": 8, + "rcf_total_updates": 1994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "dIyavWhmSYWGz65b4u-lpQ", + "detector_task_slots": 1 + } +} +``` + +The `total_entities` parameter shows you the total number of entities including the number of category fields for a detector. + +Getting the total count of entities is an expensive operation for real-time analysis of a detector with more than one category field. By default, for a real-time detection profile, a detector counts the number of entities up to a value of 10,000. For historical analysis, the anomaly detection plugin only detects the top 1,000 entities by default and caches the top entities in memory, so it doesn't cost much to get the total count of entities for historical analysis. + +The `profile` operation also provides information about each entity, such as the entity’s `last_sample_timestamp` and `last_active_timestamp`. `last_sample_timestamp` shows the last document in the input data source index containing the entity, while `last_active_timestamp` shows the timestamp when the entity’s model was last seen in the model cache. + +If there are no anomaly results for an entity, either the entity doesn't have any sample data or its model is removed from the model cache. + +#### Request + +```json +GET _plugins/_anomaly_detection/detectors//_profile?_all=true +{ + "entity": [ + { + "name": "host", + "value": "i-00f28ec1eb8997686" + } + ] +} +``` + +#### Sample Responses + +```json +{ + "category_field": "host", + "value": "i-00f28ec1eb8997686", + "is_active": true, + "last_active_timestamp": 1604026394879, + "last_sample_timestamp": 1604026394879, + "init_progress": { + "percentage": "100%" + }, + "model": { + "model_id": "TFUdd3UBBwIAGQeRh5IS_entity_i-00f28ec1eb8997686", + "model_size_in_bytes": 712480, + "node_id": "MQ-bTBW3Q2uU_2zX3pyEQg" + }, + "state": "RUNNING" +} +``` + +To get profile information for only historical analysis, specify `ad_task`. +Specifying `_all` is an expensive operation for multi-category high cardinality detectors. + +#### Request + +```json +GET _plugins/_anomaly_detection/detectors//_profile?_all +GET _plugins/_anomaly_detection/detectors//_profile/ad_task +``` + +#### Sample Responses + +```json +{ + "ad_task": { + "ad_task": { + "task_id": "CHI0TnwBYwCbWecgqgRA", + "last_update_time": 1633399648413, + "started_by": "admin", + "state": "RUNNING", + "detector_id": "3Dh6TXwBwf_U8gjURE0F", + "task_progress": 0, + "init_progress": 0, + "execution_start_time": 1633399646784, + "is_latest": true, + "task_type": "HISTORICAL_HC_DETECTOR", + "coordinating_node": "2Z4q22BySEyzakYt_A0A2A", + "detector": { + "name": "testhc-mc", + "description": "test", + "time_field": "timestamp", + "indices": [ + "server_log" + ], + "filter_query": { + "match_all": { + "boost": 1 + } + }, + "detection_interval": { + "period": { + "interval": 5, + "unit": "Minutes" + } + }, + "window_delay": { + "period": { + "interval": 1, + "unit": "Minutes" + } + }, + "shingle_size": 8, + "schema_version": 0, + "feature_attributes": [ + { + "feature_id": "2zh6TXwBwf_U8gjUQ039", + "feature_name": "test", + "feature_enabled": true, + "aggregation_query": { + "test": { + "sum": { + "field": "value" + } + } + } + } + ], + "ui_metadata": { + "features": { + "test": { + "aggregationBy": "sum", + "aggregationOf": "value", + "featureType": "simple_aggs" + } + }, + "filters": [] + }, + "last_update_time": 1633387430916, + "category_field": [ + "ip", + "error_type" + ], + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + }, + "detector_type": "MULTI_ENTITY" + }, + "detection_date_range": { + "start_time": 1632793800000, + "end_time": 1633398600000 + }, + "user": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": "__user__" + } + }, + "node_id": "2Z4q22BySEyzakYt_A0A2A", + "task_id": "CHI0TnwBYwCbWecgqgRA", + "task_type": "HISTORICAL_HC_DETECTOR", + "detector_task_slots": 10, + "total_entities_count": 32, + "pending_entities_count": 22, + "running_entities_count": 10, + "running_entities" : [ + """[{"name":"ip","value":"192.168.1.1"},{"name":"error_type","value":"error9"}]""", + ... + ], + "entity_task_profiles": [ + { + "shingle_size": 8, + "rcf_total_updates": 994, + "threshold_model_trained": true, + "threshold_model_training_data_size": 0, + "model_size_in_bytes": 1593240, + "node_id": "2Z4q22BySEyzakYt_A0A2A", + "entity": [ + { + "name": "ip", + "value": "192.168.1.1" + }, + { + "name": "error_type", + "value": "error6" + } + ], + "task_id": "9XI0TnwBYwCbWecgsAd6", + "task_type": "HISTORICAL_HC_ENTITY" + }, + ... + ] + } } ``` @@ -1718,7 +2918,6 @@ DELETE _plugins/_anomaly_detection/detectors/results } ``` - #### Sample response ```json @@ -1743,486 +2942,6 @@ DELETE _plugins/_anomaly_detection/detectors/results } ``` - ---- - -## Search detector -Introduced 1.0 -{: .label .label-purple } - -Returns all anomaly detectors for a search query. - -#### Request - -```json -GET _plugins/_anomaly_detection/detectors/_search -POST _plugins/_anomaly_detection/detectors/_search -{ - "query": { - "bool": { - "filter": [ - { - "terms": { - "indices": [ - "server_log" - ] - } - } - ] - } - } -} -``` - -#### Sample response - -```json -{ - "took": 2, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": { - "value": 1, - "relation": "eq" - }, - "max_score": 0, - "hits": [ - { - "_index": ".opensearch-anomaly-detectors", - "_type": "_doc", - "_id": "rlDtOHwBD5tpxlbyW7Nt", - "_version": 1, - "_seq_no": 0, - "_primary_term": 1, - "_score": 0, - "_source": { - "name": "test-detector", - "description": "test", - "time_field": "timestamp", - "indices": [ - "server_log" - ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 5, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "7VDtOHwBD5tpxlbyWqPs", - "feature_name": "test-feature", - "feature_enabled": true, - "aggregation_query": { - "test_feature": { - "sum": { - "field": "value" - } - } - } - } - ], - "last_update_time": 1633042652012, - "category_field": [ - "type" - ], - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "own_index", - "all_access" - ], - "custom_attribute_names": [], - "user_requested_tenant": null - }, - "detector_type": "MULTI_ENTITY" - } - } - ] - } -} -``` - ---- - -## Get detector stats -Introduced 1.0 -{: .label .label-purple } - -Provides information about how the plugin is performing. - -#### Request - -```json -GET _plugins/_anomaly_detection/stats -GET _plugins/_anomaly_detection//stats -GET _plugins/_anomaly_detection//stats/ -GET _plugins/_anomaly_detection/stats/ -``` - -#### Sample response - -```json -{ - "anomaly_detectors_index_status": "yellow", - "anomaly_detection_state_status": "yellow", - "single_entity_detector_count": 0, - "detector_count": 1, - "multi_entity_detector_count": 0, - "anomaly_detection_job_index_status": "yellow", - "models_checkpoint_index_status": "yellow", - "anomaly_results_index_status": "yellow", - "nodes": { - "hhfW2ZNVTJCtbs8rO-nF4g": { - "ad_execute_request_count": 6, - "models": [ - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578975, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_ic43SHH11NWr32xXgjRFwg", - "last_checkpoint_time": 1629827339211, - "entity": [ - { - "name": "host", - "value": "server_2" - }, - { - "name": "service", - "value": "app_6" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578979, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_OIsxy2nXMVdngK6Vv3X0uw", - "last_checkpoint_time": 1629827342444, - "entity": [ - { - "name": "host", - "value": "server_2" - }, - { - "name": "service", - "value": "app_2" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578978, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_I0L8K8ktyVnyL59CVFCLVQ", - "last_checkpoint_time": 1629827342068, - "entity": [ - { - "name": "host", - "value": "server_1" - }, - { - "name": "service", - "value": "app_4" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578975, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_ZoNYVJsq5ry6e-SWXmAt1Q", - "last_checkpoint_time": 1629827339435, - "entity": [ - { - "name": "host", - "value": "server_1" - }, - { - "name": "service", - "value": "app_6" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578978, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_qo2ANH_NS7Bg8iV4AJpHOw", - "last_checkpoint_time": 1629827341187, - "entity": [ - { - "name": "host", - "value": "server_3" - }, - { - "name": "service", - "value": "app_0" - } - ] - }, - { - "detector_id": "mmZFeXsB7JcKN0mdnMf4", - "model_type": "entity", - "last_used_time": 1629827578980, - "model_id": "mmZFeXsB7JcKN0mdnMf4_entity_412FoQwCykWTAhjVfDGQDg", - "last_checkpoint_time": 1629827342983, - "entity": [ - { - "name": "host", - "value": "server_1" - }, - { - "name": "service", - "value": "app_2" - } - ] - } - ], - "ad_canceled_batch_task_count": 0, - "ad_hc_execute_request_count": 6, - "ad_hc_execute_failure_count": 0, - "model_count": 21, - "ad_execute_failure_count": 0, - "ad_batch_task_failure_count": 0, - "ad_total_batch_task_execution_count": 0, - "ad_executing_batch_task_count": 0 - } - } -} -``` - -The `model_count` parameter shows the total number of models running on each node’s memory. -For historical analysis, you see the values for the following fields: - -- `ad_total_batch_task_execution_count` -- `ad_executing_batch_task_count` -- `ad_canceled_batch_task_count` -- `ad_batch_task_failure_count` - -If haven't run any historical analysis, these values show up as 0. - -#### Sample response - -```json -{ - "anomaly_detectors_index_status": "green", - "anomaly_detection_state_status": "green", - "single_entity_detector_count": 0, - "detector_count": 1, - "multi_entity_detector_count": 1, - "anomaly_detection_job_index_status": "green", - "models_checkpoint_index_status": "green", - "anomaly_results_index_status": "green", - "nodes": { - "bCtWtxWPThq0BIn5P5I4Xw": { - "ad_execute_request_count": 0, - "models": [ - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152729, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error20", - "last_checkpoint_time": 1633043556222, - "entity": [ - { - "name": "type", - "value": "error20" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152767, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error31", - "last_checkpoint_time": 1633043855146, - "entity": [ - { - "name": "type", - "value": "error31" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152732, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error16", - "last_checkpoint_time": 1633043557253, - "entity": [ - { - "name": "type", - "value": "error16" - } - ] - } - ], - "ad_canceled_batch_task_count": 0, - "ad_hc_execute_request_count": 0, - "ad_hc_execute_failure_count": 0, - "model_count": 8, - "ad_execute_failure_count": 0, - "ad_batch_task_failure_count": 0, - "ad_total_batch_task_execution_count": 15, - "ad_executing_batch_task_count": 3 - }, - "dIyavWhmSYWGz65b4u-lpQ": { - "ad_execute_request_count": 0, - "models": [ - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152729, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error3", - "last_checkpoint_time": 1633043256013, - "entity": [ - { - "name": "type", - "value": "error3" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152727, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error1", - "last_checkpoint_time": 1633043254819, - "entity": [ - { - "name": "type", - "value": "error1" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152735, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error22", - "last_checkpoint_time": 1633043557023, - "entity": [ - { - "name": "type", - "value": "error22" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152763, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error29", - "last_checkpoint_time": 1633043855226, - "entity": [ - { - "name": "type", - "value": "error29" - } - ] - } - ], - "ad_canceled_batch_task_count": 0, - "ad_hc_execute_request_count": 0, - "ad_hc_execute_failure_count": 0, - "model_count": 11, - "ad_execute_failure_count": 0, - "ad_batch_task_failure_count": 0, - "ad_total_batch_task_execution_count": 14, - "ad_executing_batch_task_count": 3 - }, - "2hEGbUw6ShaiKe05n_xLdA": { - "ad_execute_request_count": 5, - "models": [ - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152714, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error5", - "last_checkpoint_time": 1633043256689, - "entity": [ - { - "name": "type", - "value": "error5" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152720, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error15", - "last_checkpoint_time": 1633043553786, - "entity": [ - { - "name": "type", - "value": "error15" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152724, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error17", - "last_checkpoint_time": 1633043554909, - "entity": [ - { - "name": "type", - "value": "error17" - } - ] - }, - { - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "model_type": "entity", - "last_used_time": 1633044152751, - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error27", - "last_checkpoint_time": 1633043855105, - "entity": [ - { - "name": "type", - "value": "error27" - } - ] - } - ], - "ad_canceled_batch_task_count": 0, - "ad_hc_execute_request_count": 5, - "ad_hc_execute_failure_count": 0, - "model_count": 13, - "ad_execute_failure_count": 0, - "ad_batch_task_failure_count": 0, - "ad_total_batch_task_execution_count": 14, - "ad_executing_batch_task_count": 3 - } - } -} -``` - --- ## Create monitor @@ -2444,899 +3163,3 @@ POST _plugins/_alerting/monitors ``` --- - -## Profile detector -Introduced 1.0 -{: .label .label-purple } - -Returns information related to the current state of the detector and memory usage, including current errors and shingle size, to help troubleshoot the detector. - -This command helps locate logs by identifying the nodes that run the anomaly detector job for each detector. - -It also helps track the initialization percentage, the required shingles, and the estimated time left. - -#### Request - -```json -GET _plugins/_anomaly_detection/detectors//_profile/ -GET _plugins/_anomaly_detection/detectors//_profile?_all=true -GET _plugins/_anomaly_detection/detectors//_profile/ -GET _plugins/_anomaly_detection/detectors//_profile/, -``` - -#### Sample Responses - -```json -GET _plugins/_anomaly_detection/detectors//_profile -{ - "state": "DISABLED", - "error": "Stopped detector: AD models memory usage exceeds our limit." -} - -GET _plugins/_anomaly_detection/detectors//_profile?_all=true&pretty -{ - "state": "RUNNING", - "error": "", - "models": [ - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error20", - "entity": [ - { - "name": "type", - "value": "error20" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error31", - "entity": [ - { - "name": "type", - "value": "error31" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error21", - "entity": [ - { - "name": "type", - "value": "error21" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error13", - "entity": [ - { - "name": "type", - "value": "error13" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error24", - "entity": [ - { - "name": "type", - "value": "error24" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error25", - "entity": [ - { - "name": "type", - "value": "error25" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error28", - "entity": [ - { - "name": "type", - "value": "error28" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error16", - "entity": [ - { - "name": "type", - "value": "error16" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error3", - "entity": [ - { - "name": "type", - "value": "error3" - } - ], - "model_size_in_bytes": 403491, - "node_id": "dIyavWhmSYWGz65b4u-lpQ" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error27", - "entity": [ - { - "name": "type", - "value": "error27" - } - ], - "model_size_in_bytes": 403491, - "node_id": "2hEGbUw6ShaiKe05n_xLdA" - } - ], - "total_size_in_bytes": 12911712, - "init_progress": { - "percentage": "100%" - }, - "total_entities": 33, - "active_entities": 32, - "ad_task": { - "ad_task": { - "task_id": "Os4HOXwBCi2h__AONgpc", - "last_update_time": 1633044347855, - "started_by": "admin", - "state": "RUNNING", - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "task_progress": 0, - "init_progress": 0, - "execution_start_time": 1633044346460, - "is_latest": true, - "task_type": "HISTORICAL_HC_DETECTOR", - "coordinating_node": "2hEGbUw6ShaiKe05n_xLdA", - "detector": { - "name": "test-detector", - "description": "test", - "time_field": "timestamp", - "indices": [ - "server_log" - ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 5, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "7VDtOHwBD5tpxlbyWqPs", - "feature_name": "test-feature", - "feature_enabled": true, - "aggregation_query": { - "test_feature": { - "sum": { - "field": "value" - } - } - } - } - ], - "ui_metadata": { - "features": { - "test-feature": { - "aggregationBy": "sum", - "aggregationOf": "value", - "featureType": "simple_aggs" - } - }, - "filters": [] - }, - "last_update_time": 1633042652012, - "category_field": [ - "type" - ], - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "own_index", - "all_access" - ], - "custom_attribute_names": [], - "user_requested_tenant": null - }, - "detector_type": "MULTI_ENTITY" - }, - "detection_date_range": { - "start_time": 1632437820000, - "end_time": 1633042620000 - }, - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "own_index", - "all_access" - ], - "custom_attribute_names": [], - "user_requested_tenant": "__user__" - } - }, - "node_id": "2hEGbUw6ShaiKe05n_xLdA", - "task_id": "Os4HOXwBCi2h__AONgpc", - "task_type": "HISTORICAL_HC_DETECTOR", - "detector_task_slots": 10, - "total_entities_count": 32, - "pending_entities_count": 22, - "running_entities_count": 10, - "running_entities": [ - "error9", - "error8", - "error7", - "error6", - "error5", - "error4", - "error32", - "error31", - "error30", - "error3" - ], - "entity_task_profiles": [ - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw", - "entity": [ - { - "name": "type", - "value": "error6" - } - ], - "task_id": "P84HOXwBCi2h__AOOgrC", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "dIyavWhmSYWGz65b4u-lpQ", - "entity": [ - { - "name": "type", - "value": "error4" - } - ], - "task_id": "Kc4HOXwBCi2h__AOOw6Y", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "dIyavWhmSYWGz65b4u-lpQ", - "entity": [ - { - "name": "type", - "value": "error8" - } - ], - "task_id": "Pc4HOXwBCi2h__AOOgqJ", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "2hEGbUw6ShaiKe05n_xLdA", - "entity": [ - { - "name": "type", - "value": "error7" - } - ], - "task_id": "Ps4HOXwBCi2h__AOOgqh", - "task_type": "HISTORICAL_HC_ENTITY" - } - ] - }, - "model_count": 32 -} - -GET _plugins/_anomaly_detection/detectors//_profile/total_size_in_bytes -{ - "total_size_in_bytes": 13369344 -} -``` - -You can see the `ad_task` field only for historical analysis. - -The `model_count` parameter shows the total number of models that a detector runs on each node’s memory. This is useful if you have several models running on your cluster and want to know the count. - -If you configured the category field, you can see the number of unique values in the field and all active entities with models running in memory. - -You can use this data to estimate how much memory is required for anomaly detection so you can decide how to size your cluster. For example, if a detector has one million entities and only 10 of them are active in memory, you need to scale your cluster up or out. - -#### Request - -```json -GET _plugins/_anomaly_detection/detectors//_profile?_all=true&pretty -{ - "state": "RUNNING", - "error": "", - "models": [ - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error20", - "entity": [ - { - "name": "type", - "value": "error20" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error13", - "entity": [ - { - "name": "type", - "value": "error13" - } - ], - "model_size_in_bytes": 403491, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw" - }, - { - "model_id": "rlDtOHwBD5tpxlbyW7Nt_entity_error27", - "entity": [ - { - "name": "type", - "value": "error27" - } - ], - "model_size_in_bytes": 403491, - "node_id": "2hEGbUw6ShaiKe05n_xLdA" - } - ], - "total_size_in_bytes": 12911712, - "init_progress": { - "percentage": "100%" - }, - "total_entities": 33, - "active_entities": 32, - "ad_task": { - "ad_task": { - "task_id": "Os4HOXwBCi2h__AONgpc", - "last_update_time": 1633044347855, - "started_by": "admin", - "state": "RUNNING", - "detector_id": "rlDtOHwBD5tpxlbyW7Nt", - "task_progress": 0, - "init_progress": 0, - "execution_start_time": 1633044346460, - "is_latest": true, - "task_type": "HISTORICAL_HC_DETECTOR", - "coordinating_node": "2hEGbUw6ShaiKe05n_xLdA", - "detector": { - "name": "test-detector", - "description": "test", - "time_field": "timestamp", - "indices": [ - "server_log" - ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 5, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "7VDtOHwBD5tpxlbyWqPs", - "feature_name": "test-feature", - "feature_enabled": true, - "aggregation_query": { - "test_feature": { - "sum": { - "field": "value" - } - } - } - } - ], - "ui_metadata": { - "features": { - "test-feature": { - "aggregationBy": "sum", - "aggregationOf": "value", - "featureType": "simple_aggs" - } - }, - "filters": [] - }, - "last_update_time": 1633042652012, - "category_field": [ - "type" - ], - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "own_index", - "all_access" - ], - "custom_attribute_names": [], - "user_requested_tenant": null - }, - "detector_type": "MULTI_ENTITY" - }, - "detection_date_range": { - "start_time": 1632437820000, - "end_time": 1633042620000 - }, - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "own_index", - "all_access" - ], - "custom_attribute_names": [], - "user_requested_tenant": "__user__" - } - }, - "node_id": "2hEGbUw6ShaiKe05n_xLdA", - "task_id": "Os4HOXwBCi2h__AONgpc", - "task_type": "HISTORICAL_HC_DETECTOR", - "detector_task_slots": 10, - "total_entities_count": 32, - "pending_entities_count": 22, - "running_entities_count": 10, - "running_entities": [ - "error9", - "error8", - "error7", - "error6", - "error5", - "error4", - "error32", - "error31", - "error30", - "error3" - ], - "entity_task_profiles": [ - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw", - "entity": [ - { - "name": "type", - "value": "error6" - } - ], - "task_id": "P84HOXwBCi2h__AOOgrC", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw", - "entity": [ - { - "name": "type", - "value": "error5" - } - ], - "task_id": "QM4HOXwBCi2h__AOOgre", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "bCtWtxWPThq0BIn5P5I4Xw", - "entity": [ - { - "name": "type", - "value": "error9" - } - ], - "task_id": "PM4HOXwBCi2h__AOOgp3", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "2hEGbUw6ShaiKe05n_xLdA", - "entity": [ - { - "name": "type", - "value": "error3" - } - ], - "task_id": "Fs4HOXwBCi2h__AOPBLH", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "2hEGbUw6ShaiKe05n_xLdA", - "entity": [ - { - "name": "type", - "value": "error32" - } - ], - "task_id": "Ks4HOXwBCi2h__AOOw7D", - "task_type": "HISTORICAL_HC_ENTITY" - }, - { - "shingle_size": 8, - "rcf_total_updates": 994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "2hEGbUw6ShaiKe05n_xLdA", - "entity": [ - { - "name": "type", - "value": "error7" - } - ], - "task_id": "Ps4HOXwBCi2h__AOOgqh", - "task_type": "HISTORICAL_HC_ENTITY" - } - ] - }, - "model_count": 32 -} -``` - -For a single-entity detector: - -#### Sample response - -```json -{ - "state": "INIT", - "total_size_in_bytes": 0, - "init_progress": { - "percentage": "0%", - "needed_shingles": 128 - }, - "ad_task": { - "ad_task": { - "task_id": "cfUNOXwBFLNqSEcxAlde", - "last_update_time": 1633044731640, - "started_by": "admin", - "state": "RUNNING", - "detector_id": "qL4NOXwB__6eNorTAKtJ", - "task_progress": 0.49603173, - "init_progress": 1, - "current_piece": 1632739800000, - "execution_start_time": 1633044726365, - "is_latest": true, - "task_type": "HISTORICAL_SINGLE_ENTITY", - "coordinating_node": "bCtWtxWPThq0BIn5P5I4Xw", - "worker_node": "dIyavWhmSYWGz65b4u-lpQ", - "detector": { - "name": "detector1", - "description": "test", - "time_field": "timestamp", - "indices": [ - "server_log" - ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 5, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "p74NOXwB__6eNorTAKss", - "feature_name": "test-feature", - "feature_enabled": true, - "aggregation_query": { - "test_feature": { - "sum": { - "field": "value" - } - } - } - } - ], - "ui_metadata": { - "features": { - "test-feature": { - "aggregationBy": "sum", - "aggregationOf": "value", - "featureType": "simple_aggs" - } - }, - "filters": [] - }, - "last_update_time": 1633044725832, - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "own_index", - "all_access" - ], - "custom_attribute_names": [], - "user_requested_tenant": "__user__" - }, - "detector_type": "SINGLE_ENTITY" - }, - "detection_date_range": { - "start_time": 1632439925885, - "end_time": 1633044725885 - }, - "user": { - "name": "admin", - "backend_roles": [ - "admin" - ], - "roles": [ - "own_index", - "all_access" - ], - "custom_attribute_names": [], - "user_requested_tenant": "__user__" - } - }, - "shingle_size": 8, - "rcf_total_updates": 1994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "model_size_in_bytes": 1593240, - "node_id": "dIyavWhmSYWGz65b4u-lpQ", - "detector_task_slots": 1 - } -} -``` - -The `total_entities` parameter shows you the total number of entities including the number of category fields for a detector. - -Getting the total count of entities is an expensive operation for real-time analysis of a detector with more than one category field. By default, for a real-time detection profile, a detector counts the number of entities up to a value of 10,000. For historical analysis, the anomaly detection plugin only detects the top 1,000 entities by default and caches the top entities in memory to reduce the cost of getting the total count of entities for historical analysis. - -The `profile` operation also provides information about each entity, such as the entity’s `last_sample_timestamp` and `last_active_timestamp`. `last_sample_timestamp` shows the last document in the input data source index containing the entity, while `last_active_timestamp` shows the timestamp when the entity’s model was last seen in the model cache. - -If there are no anomaly results for an entity, either the entity doesn't have any sample data or its model is removed from the model cache. - - -#### Request - -```json -GET _plugins/_anomaly_detection/detectors//_profile?_all=true -{ - "entity": [ - { - "name": "host", - "value": "i-00f28ec1eb8997686" - } - ] -} -``` - -#### Sample Responses - -```json -{ - "category_field": "host", - "value": "i-00f28ec1eb8997686", - "is_active": true, - "last_active_timestamp": 1604026394879, - "last_sample_timestamp": 1604026394879, - "init_progress": { - "percentage": "100%" - }, - "model": { - "model_id": "TFUdd3UBBwIAGQeRh5IS_entity_i-00f28ec1eb8997686", - "model_size_in_bytes": 712480, - "node_id": "MQ-bTBW3Q2uU_2zX3pyEQg" - }, - "state": "RUNNING" -} -``` - -To get profile information for only historical analysis, specify `ad_task`. -Specifying `_all` is an expensive operation for multi-category high cardinality detectors. - -#### Request - -```json -GET _plugins/_anomaly_detection/detectors//_profile?_all -GET _plugins/_anomaly_detection/detectors//_profile/ad_task -``` - -#### Sample Responses - -```json -{ - "ad_task": { - "ad_task": { - "task_id": "JXxyG3YBv5IHYYfMlFS2", - "last_update_time": 1606778263543, - "state": "STOPPED", - "detector_id": "SwvxCHYBPhugfWD9QAL6", - "task_progress": 0.010480972, - "init_progress": 1, - "current_piece": 1578140400000, - "execution_start_time": 1606778262709, - "is_latest": true, - "task_type": "HISTORICAL", - "detector": { - "name": "historical_test1", - "description": "test", - "time_field": "timestamp", - "indices": [ - "server_log" - ], - "filter_query": { - "match_all": { - "boost": 1 - } - }, - "detection_interval": { - "period": { - "interval": 5, - "unit": "Minutes" - } - }, - "window_delay": { - "period": { - "interval": 1, - "unit": "Minutes" - } - }, - "shingle_size": 8, - "schema_version": 0, - "feature_attributes": [ - { - "feature_id": "zgvyCHYBPhugfWD9Ap_F", - "feature_name": "sum", - "feature_enabled": true, - "aggregation_query": { - "sum": { - "sum": { - "field": "value" - } - } - } - }, - { - "feature_id": "zwvyCHYBPhugfWD9Ap_G", - "feature_name": "max", - "feature_enabled": true, - "aggregation_query": { - "max": { - "max": { - "field": "value" - } - } - } - } - ], - "ui_metadata": { - "features": { - "max": { - "aggregationBy": "max", - "aggregationOf": "value", - "featureType": "simple_aggs" - }, - "sum": { - "aggregationBy": "sum", - "aggregationOf": "value", - "featureType": "simple_aggs" - } - }, - "filters": [], - "filterType": "simple_filter" - }, - "last_update_time": 1606467935713, - "detector_type": "HISTORICAL_SIGLE_ENTITY", - "detection_date_range": { - "start_time": 1577840400000, - "end_time": 1606463775000 - } - } - }, - "shingle_size": 8, - "rcf_total_updates": 1994, - "threshold_model_trained": true, - "threshold_model_training_data_size": 0, - "node_id": "Q9yznwxvTz-yJxtz7rJlLg" - } -} -``` - ---- diff --git a/_monitoring-plugins/ad/index.md b/_monitoring-plugins/ad/index.md index d41f6455..6abfe5ac 100644 --- a/_monitoring-plugins/ad/index.md +++ b/_monitoring-plugins/ad/index.md @@ -79,7 +79,7 @@ To get the entity model size of a detector, use the [profile detector API]({{sit This formula provides a good starting point, but make sure to test with a representative workload. {: .note } -For example, for a cluster with 3 data nodes, each with 8G of JVM heap size, a maximum memory percentage of 10% (default), and the entity model size of the detector as 1MB: the total number of unique entities supported is (8.096 * 10^9 * 0.1 / 1M ) * 3 = 2429. +For example, for a cluster with three data nodes, each with 8 GB of JVM heap size, a maximum memory percentage of 10% (default), and the entity model size of the detector as 1MB: the total number of unique entities supported is (8.096 * 10^9 * 0.1 / 1 MB ) * 3 = 2429. If you set the total number of unique entities higher than this number that you calculate (in this case: 2429), the anomaly detector makes its best effort to model the extra entities. The detector prioritizes entities that occur more often and are more recent. diff --git a/_monitoring-plugins/ad/settings.md b/_monitoring-plugins/ad/settings.md index 5aa73dc4..e63ab922 100644 --- a/_monitoring-plugins/ad/settings.md +++ b/_monitoring-plugins/ad/settings.md @@ -40,7 +40,7 @@ Setting | Default | Description `plugins.anomaly_detection.batch_task_piece_size` | 1,000 | The date range for a historical task is split into smaller pieces and the anomaly detection plugin runs the task piece by piece. Each piece contains 1,000 detection intervals by default. For example, if detector interval is 1 minute and one piece is 1,000 minutes, the feature data is queried every 1,000 minutes. You can change this setting from 1 to 10,000. `plugins.anomaly_detection.batch_task_piece_interval_seconds` | 5 | Add a time interval between two pieces of the same historical analysis task. This interval prevents the task from consuming too much of the available resources and starving other operations like search and bulk index. You can change this setting from 1 to 600 seconds. `plugins.anomaly_detection.max_top_entities_for_historical_analysis` | 1,000 | The maximum number of top entities that you run for a high cardinality detector historical analysis. The range is from 1 to 10,000. -`plugins.anomaly_detection.max_running_entities_per_detector_for_historical_analysis` | 10 | The number of entity tasks that you can run in parallel for a single high cardinality detector. The task slots available on your cluster also impact how many entities run in parallel. If a cluster has 3 data nodes, each data node has 10 task slots by default. Say you already have two high cardinality detectors and each of them run 10 entities. If you start a single-flow detector that takes 1 task slot, the number of task slots available is 10 * 3 - 10 * 2 - 1 = 9. if you now start a new high cardinality detector, the detector can only run 9 entities in parallel and not 10. You can tune this value from 1 to 1,000 based on your cluster's capability. If you set a higher value, the anomaly detection plugin runs historical analysis faster but also consumes more resources. +`plugins.anomaly_detection.max_running_entities_per_detector_for_historical_analysis` | 10 | The number of entity tasks that you can run in parallel for a single high cardinality detector. The task slots available on your cluster also impact how many entities run in parallel. If a cluster has 3 data nodes, each data node has 10 task slots by default. Say you already have two high cardinality detectors and each of them run 10 entities. If you start a single-flow detector that takes 1 task slot, the number of task slots available is 10 * 3 - 10 * 2 - 1 = 9. If you now start a new high cardinality detector, the detector can only run 9 entities in parallel and not 10. You can tune this value from 1 to 1,000 based on your cluster's capability. If you set a higher value, the anomaly detection plugin runs historical analysis faster but also consumes more resources. `plugins.anomaly_detection.max_cached_deleted_tasks` | 1,000 | You can rerun historical analysis for a single detector as many times as you like. The anomaly detection plugin only keeps a limited number of old tasks, by default 1 old task. If you run historical analysis three times for a detector, the oldest task is deleted. Because historical analysis generates a number of anomaly results in a short span of time, it's necessary to clean up anomaly results for a deleted task. With this field, you can configure how many deleted tasks you can cache at most. The plugin cleans up a task's results when it's deleted. If the plugin fails to do this cleanup, it adds the task's results into a cache and an hourly cron job performs the cleanup. You can use this setting to limit how many old tasks are put into cache to avoid a DDoS attack. After an hour, if still you find an old task result in the cache, use the [delete detector results API]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/api/#delete-detector-results) to delete the task result manually. You can tune this setting from 1 to 10,000. `plugins.anomaly_detection.delete_anomaly_result_when_delete_detector` | False | Whether the anomaly detection plugin deletes the anomaly result when you delete a detector. If you want to save some disk space, especially if you've high cardinality detectors generating a lot of results, set this field to true. Alternatively, you can use the [delete detector results API]({{site.url}}{{site.baseurl}}/monitoring-plugins/ad/api/#delete-detector-results) to manually delete the results. `plugins.anomaly_detection.dedicated_cache_size` | 10 | If the real-time analysis of a high cardinality detector starts successfully, the anomaly detection plugin guarantees keeping 10 (dynamically adjustable via this setting) entities' models in memory per node. If the number of entities exceeds this limit, the plugin puts the extra entities' models in a memory space shared by all detectors. The actual number of entities varies based on the memory that you've available and the frequencies of the entities. If you'd like the plugin to guarantee keeping more entities' models in memory and if you're cluster has sufficient memory, you can increase this setting value.