mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-03-25 01:19:02 +00:00
[DOCS] Enabled more ML code snippet testing (elastic/x-pack-elasticsearch#3764)
Original commit: elastic/x-pack-elasticsearch@518dce3ddd
This commit is contained in:
Lisa Cawley
GitHub
parent
d2dc630e84
commit
4e0c1d1b60
@ -9,7 +9,6 @@ apply plugin: 'elasticsearch.docs-test'
|
||||
* only remove entries from this list. When it is empty we'll remove it
|
||||
* entirely and have a party! There will be cake and everything.... */
|
||||
buildRestTests.expectedUnconvertedCandidates = [
|
||||
'en/ml/getting-started-data.asciidoc',
|
||||
'en/ml/functions/count.asciidoc',
|
||||
'en/ml/functions/geo.asciidoc',
|
||||
'en/ml/functions/info.asciidoc',
|
||||
@ -65,8 +64,6 @@ buildRestTests.expectedUnconvertedCandidates = [
|
||||
'en/watcher/trigger/schedule/yearly.asciidoc',
|
||||
'en/watcher/troubleshooting.asciidoc',
|
||||
'en/ml/api-quickref.asciidoc',
|
||||
'en/rest-api/ml/close-job.asciidoc',
|
||||
'en/rest-api/ml/delete-datafeed.asciidoc',
|
||||
'en/rest-api/ml/delete-snapshot.asciidoc',
|
||||
'en/rest-api/ml/flush-job.asciidoc',
|
||||
'en/rest-api/ml/forecast.asciidoc',
|
||||
@ -81,10 +78,8 @@ buildRestTests.expectedUnconvertedCandidates = [
|
||||
'en/rest-api/ml/update-job.asciidoc',
|
||||
'en/rest-api/ml/update-snapshot.asciidoc',
|
||||
'en/rest-api/ml/validate-detector.asciidoc',
|
||||
'en/rest-api/ml/delete-job.asciidoc',
|
||||
'en/rest-api/ml/get-datafeed.asciidoc',
|
||||
'en/rest-api/ml/get-influencer.asciidoc',
|
||||
'en/rest-api/ml/get-job.asciidoc',
|
||||
'en/rest-api/ml/get-snapshot.asciidoc',
|
||||
'en/rest-api/ml/revert-snapshot.asciidoc',
|
||||
'en/rest-api/ml/validate-job.asciidoc',
|
||||
@ -301,21 +296,7 @@ setups['server_metrics_index'] = '''
|
||||
total:
|
||||
type: long
|
||||
'''
|
||||
setups['server_metrics_job'] = '''
|
||||
- do:
|
||||
indices.create:
|
||||
index: server-metrics
|
||||
body:
|
||||
settings:
|
||||
number_of_shards: 1
|
||||
number_of_replicas: 0
|
||||
mappings:
|
||||
metric:
|
||||
properties:
|
||||
timestamp:
|
||||
type: date
|
||||
total:
|
||||
type: long
|
||||
setups['server_metrics_job'] = setups['server_metrics_index'] + '''
|
||||
- do:
|
||||
xpack.ml.put_job:
|
||||
job_id: "total-requests"
|
||||
@ -337,7 +318,7 @@ setups['server_metrics_job'] = '''
|
||||
}
|
||||
}
|
||||
'''
|
||||
setups['server_metrics_openjob'] = setups['server_metrics_job'] + '''
|
||||
setups['server_metrics_datafeed'] = setups['server_metrics_job'] + '''
|
||||
- do:
|
||||
xpack.ml.put_datafeed:
|
||||
datafeed_id: "datafeed-total-requests"
|
||||
@ -347,6 +328,8 @@ setups['server_metrics_openjob'] = setups['server_metrics_job'] + '''
|
||||
"indexes":"server-metrics",
|
||||
"types":"metric"
|
||||
}
|
||||
'''
|
||||
setups['server_metrics_openjob'] = setups['server_metrics_datafeed'] + '''
|
||||
- do:
|
||||
xpack.ml.open_job:
|
||||
job_id: "total-requests"
|
||||
|
||||
@ -50,7 +50,7 @@ https://download.elastic.co/demos/machine_learning/gettingstarted/server_metrics
|
||||
|
||||
Use the following commands to extract the files:
|
||||
|
||||
[source,shell]
|
||||
[source,sh]
|
||||
----------------------------------
|
||||
tar -zxvf server_metrics.tar.gz
|
||||
----------------------------------
|
||||
@ -77,6 +77,7 @@ Each document in the server-metrics data set has the following schema:
|
||||
"total":40476
|
||||
}
|
||||
----------------------------------
|
||||
// NOTCONSOLE
|
||||
|
||||
TIP: The sample data sets include summarized data. For example, the `total`
|
||||
value is a sum of the requests that were received by a specific service at a
|
||||
@ -103,9 +104,8 @@ with your actual user ID and password.
|
||||
The script runs a command similar to the following example, which sets up a
|
||||
mapping for the data set:
|
||||
|
||||
[source,shell]
|
||||
[source,sh]
|
||||
----------------------------------
|
||||
|
||||
curl -u elastic:x-pack-test-password -X PUT -H 'Content-Type: application/json'
|
||||
http://localhost:9200/server-metrics -d '{
|
||||
"settings":{
|
||||
@ -141,27 +141,17 @@ http://localhost:9200/server-metrics -d '{
|
||||
}
|
||||
}'
|
||||
----------------------------------
|
||||
// NOTCONSOLE
|
||||
|
||||
NOTE: If you run this command, you must replace `x-pack-test-password` with your
|
||||
actual password.
|
||||
|
||||
////
|
||||
This mapping specifies the following qualities for the data set:
|
||||
|
||||
* The _@timestamp_ field is a date.
|
||||
//that uses the ISO format `epoch_second`,
|
||||
//which is the number of seconds since the epoch.
|
||||
* The _accept_, _deny_, and _total_ fields are long numbers.
|
||||
* The _host
|
||||
////
|
||||
|
||||
You can then use the {es} `bulk` API to load the data set. The
|
||||
`upload_server-metrics.sh` script runs commands similar to the following
|
||||
example, which loads the four JSON files:
|
||||
|
||||
[source,shell]
|
||||
[source,sh]
|
||||
----------------------------------
|
||||
|
||||
curl -u elastic:x-pack-test-password -X POST -H "Content-Type: application/json"
|
||||
http://localhost:9200/server-metrics/_bulk --data-binary "@server-metrics_1.json"
|
||||
|
||||
@ -174,6 +164,7 @@ http://localhost:9200/server-metrics/_bulk --data-binary "@server-metrics_3.json
|
||||
curl -u elastic:x-pack-test-password -X POST -H "Content-Type: application/json"
|
||||
http://localhost:9200/server-metrics/_bulk --data-binary "@server-metrics_4.json"
|
||||
----------------------------------
|
||||
// NOTCONSOLE
|
||||
|
||||
TIP: This will upload 200MB of data. This is split into 4 files as there is a
|
||||
maximum 100MB limit when using the `_bulk` API.
|
||||
@ -183,20 +174,20 @@ available.
|
||||
|
||||
You can verify that the data was loaded successfully with the following command:
|
||||
|
||||
[source,shell]
|
||||
[source,sh]
|
||||
----------------------------------
|
||||
|
||||
curl 'http://localhost:9200/_cat/indices?v' -u elastic:x-pack-test-password
|
||||
----------------------------------
|
||||
// NOTCONSOLE
|
||||
|
||||
You should see output similar to the following:
|
||||
|
||||
[source,shell]
|
||||
[source,txt]
|
||||
----------------------------------
|
||||
|
||||
health status index ... pri rep docs.count ...
|
||||
green open server-metrics ... 1 0 905940 ...
|
||||
----------------------------------
|
||||
// NOTCONSOLE
|
||||
|
||||
Next, you must define an index pattern for this data set:
|
||||
|
||||
|
||||
@ -73,14 +73,14 @@ For more information, see {xpack-ref}/security-privileges.html[Security Privileg
|
||||
|
||||
==== Examples
|
||||
|
||||
The following example closes the `event_rate` job:
|
||||
The following example closes the `total-requests` job:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
POST _xpack/ml/anomaly_detectors/event_rate/_close
|
||||
POST _xpack/ml/anomaly_detectors/total-requests/_close
|
||||
--------------------------------------------------
|
||||
// CONSOLE
|
||||
// TEST[skip:todo]
|
||||
// TEST[setup:server_metrics_openjob]
|
||||
|
||||
When the job is closed, you receive the following results:
|
||||
[source,js]
|
||||
@ -89,3 +89,4 @@ When the job is closed, you receive the following results:
|
||||
"closed": true
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE
|
||||
|
||||
@ -40,14 +40,14 @@ For more information, see {xpack-ref}/security-privileges.html[Security Privileg
|
||||
|
||||
==== Examples
|
||||
|
||||
The following example deletes the `datafeed-it-ops` {dfeed}:
|
||||
The following example deletes the `datafeed-total-requests` {dfeed}:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
DELETE _xpack/ml/datafeeds/datafeed-it-ops
|
||||
DELETE _xpack/ml/datafeeds/datafeed-total-requests
|
||||
--------------------------------------------------
|
||||
// CONSOLE
|
||||
// TEST[skip:todo]
|
||||
// TEST[setup:server_metrics_datafeed]
|
||||
|
||||
When the {dfeed} is deleted, you receive the following results:
|
||||
[source,js]
|
||||
@ -56,3 +56,4 @@ When the {dfeed} is deleted, you receive the following results:
|
||||
"acknowledged": true
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE
|
||||
|
||||
@ -45,19 +45,18 @@ separated list.
|
||||
|
||||
You must have `manage_ml`, or `manage` cluster privileges to use this API.
|
||||
For more information, see {xpack-ref}/security-privileges.html[Security Privileges].
|
||||
//<<privileges-list-cluster>>.
|
||||
|
||||
|
||||
==== Examples
|
||||
|
||||
The following example deletes the `event_rate` job:
|
||||
The following example deletes the `total-requests` job:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
DELETE _xpack/ml/anomaly_detectors/event_rate
|
||||
DELETE _xpack/ml/anomaly_detectors/total-requests
|
||||
--------------------------------------------------
|
||||
// CONSOLE
|
||||
// TEST[skip:todo]
|
||||
// TEST[setup:server_metrics_job]
|
||||
|
||||
When the job is deleted, you receive the following results:
|
||||
[source,js]
|
||||
@ -66,3 +65,4 @@ When the job is deleted, you receive the following results:
|
||||
"acknowledged": true
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE
|
||||
|
||||
@ -52,14 +52,14 @@ privileges to use this API. For more information, see
|
||||
|
||||
==== Examples
|
||||
|
||||
The following example gets configuration information for the `farequote` job:
|
||||
The following example gets configuration information for the `total-requests` job:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
GET _xpack/ml/anomaly_detectors/farequote
|
||||
GET _xpack/ml/anomaly_detectors/total-requests
|
||||
--------------------------------------------------
|
||||
// CONSOLE
|
||||
// TEST[skip:todo]
|
||||
// TEST[setup:server_metrics_job]
|
||||
|
||||
The API returns the following results:
|
||||
[source,js]
|
||||
@ -68,35 +68,36 @@ The API returns the following results:
|
||||
"count": 1,
|
||||
"jobs": [
|
||||
{
|
||||
"job_id": "farequote",
|
||||
"job_id": "total-requests",
|
||||
"job_type": "anomaly_detector",
|
||||
"description": "Multi-metric job",
|
||||
"create_time": 1491948149563,
|
||||
"finished_time": 1491948166289,
|
||||
"job_version": "7.0.0-alpha1",
|
||||
"description": "Total sum of requests",
|
||||
"create_time": 1517011406091,
|
||||
"analysis_config": {
|
||||
"bucket_span": "5m",
|
||||
"bucket_span": "10m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "mean(responsetime)",
|
||||
"function": "mean",
|
||||
"field_name": "responsetime",
|
||||
"partition_field_name": "airline",
|
||||
"rules": [],
|
||||
"detector_description": "Sum of total",
|
||||
"function": "sum",
|
||||
"field_name": "total",
|
||||
"rules": [ ],
|
||||
"detector_index": 0
|
||||
}
|
||||
],
|
||||
"influencers": [
|
||||
"airline"
|
||||
]
|
||||
"influencers": [ ]
|
||||
},
|
||||
"analysis_limits": {
|
||||
"model_memory_limit": "1024mb"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp",
|
||||
"time_field": "timestamp",
|
||||
"time_format": "epoch_ms"
|
||||
},
|
||||
"model_snapshot_retention_days": 1,
|
||||
"model_snapshot_id": "1491948163",
|
||||
"results_index_name": "shared"
|
||||
}
|
||||
]
|
||||
}
|
||||
----
|
||||
// TESTRESPONSE[s/"7.0.0-alpha1"/$body.$_path/]
|
||||
// TESTRESPONSE[s/1517011406091/$body.$_path/]
|
||||
|
||||
@ -40,6 +40,11 @@ so do not set the `background_persist_interval` value too low.
|
||||
`description`::
|
||||
(string) An optional description of the job.
|
||||
|
||||
`established_model_memory`::
|
||||
(long) The approximate amount of memory resources that have been used for
|
||||
analytical processing. This field is present only when the analytics have used
|
||||
a stable amount of memory for several consecutive buckets.
|
||||
|
||||
`finished_time`::
|
||||
(string) If the job closed or failed, this is the time the job finished,
|
||||
otherwise it is `null`.
|
||||
@ -56,6 +61,9 @@ so do not set the `background_persist_interval` value too low.
|
||||
`job_type`::
|
||||
(string) Reserved for future use, currently set to `anomaly_detector`.
|
||||
|
||||
`job_version`::
|
||||
(string) The version of {es} that existed on the node when the job was created.
|
||||
|
||||
`model_plot_config`::
|
||||
(object) Configuration properties for storing additional model information.
|
||||
See <<ml-apimodelplotconfig, model plot configuration>>.
|
||||
@ -217,6 +225,11 @@ Each detector has the following properties:
|
||||
`detector_description`::
|
||||
(string) A description of the detector. For example, `Low event rate`.
|
||||
|
||||
`detector_index`::
|
||||
(integer) A unique identifier for the detector. This identifier is based on
|
||||
the order of the detectors in the `analysis_config`, starting at zero. You can
|
||||
use this identifier when you want to update a specific detector.
|
||||
|
||||
`exclude_frequent`::
|
||||
(string) Contains one of the following values: `all`, `none`, `by`, or `over`.
|
||||
If set, frequent entities are excluded from influencing the anomaly results.
|
||||
@ -266,10 +279,6 @@ LEAVE UNDOCUMENTED
|
||||
(array) TBD
|
||||
////
|
||||
|
||||
`detector_index`::
|
||||
(integer) Unique ID for the detector, used when updating it.
|
||||
Based on the order of detectors within the `analysis_config`, starting at zero.
|
||||
|
||||
[float]
|
||||
[[ml-datadescription]]
|
||||
==== Data Description Objects
|
||||
|
||||
@ -104,8 +104,7 @@ PUT _xpack/ml/datafeeds/datafeed-total-requests
|
||||
{
|
||||
"job_id": "total-requests",
|
||||
"indices": ["server-metrics"],
|
||||
"types": ["metric"],
|
||||
"scroll_size": 1000
|
||||
"types": ["metric"]
|
||||
}
|
||||
--------------------------------------------------
|
||||
// CONSOLE
|
||||
@ -135,5 +134,5 @@ When the {dfeed} is created, you receive the following results:
|
||||
}
|
||||
}
|
||||
----
|
||||
//TESTRESPONSE[s/"query_delay": "83474ms"/"query_delay": $body.query_delay/]
|
||||
//TESTRESPONSE[s/"query.boost": "1.0"/"query.boost": $body.query.boost/]
|
||||
// TESTRESPONSE[s/"query_delay": "83474ms"/"query_delay": $body.query_delay/]
|
||||
// TESTRESPONSE[s/"query.boost": "1.0"/"query.boost": $body.query.boost/]
|
||||
|
||||
@ -80,28 +80,27 @@ For more information, see
|
||||
|
||||
==== Examples
|
||||
|
||||
The following example creates the `it-ops-kpi` job:
|
||||
The following example creates the `total-requests` job:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
PUT _xpack/ml/anomaly_detectors/it-ops-kpi
|
||||
PUT _xpack/ml/anomaly_detectors/total-requests
|
||||
{
|
||||
"description":"First simple job",
|
||||
"analysis_config":{
|
||||
"bucket_span": "5m",
|
||||
"latency": "0ms",
|
||||
"detectors":[
|
||||
{
|
||||
"detector_description": "low_sum(events_per_min)",
|
||||
"function":"low_sum",
|
||||
"field_name": "events_per_min"
|
||||
}
|
||||
]
|
||||
},
|
||||
"data_description": {
|
||||
"time_field":"@timestamp",
|
||||
"time_format":"epoch_ms"
|
||||
}
|
||||
"description" : "Total sum of requests",
|
||||
"analysis_config" : {
|
||||
"bucket_span":"10m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "Sum of total",
|
||||
"function": "sum",
|
||||
"field_name": "total"
|
||||
}
|
||||
]
|
||||
},
|
||||
"data_description" : {
|
||||
"time_field":"timestamp",
|
||||
"time_format": "epoch_ms"
|
||||
}
|
||||
}
|
||||
--------------------------------------------------
|
||||
// CONSOLE
|
||||
@ -110,19 +109,18 @@ When the job is created, you receive the following results:
|
||||
[source,js]
|
||||
----
|
||||
{
|
||||
"job_id": "it-ops-kpi",
|
||||
"job_id": "total-requests",
|
||||
"job_type": "anomaly_detector",
|
||||
"job_version": "7.0.0-alpha1",
|
||||
"description": "First simple job",
|
||||
"create_time": 1502832478794,
|
||||
"description": "Total sum of requests",
|
||||
"create_time": 1517011406091,
|
||||
"analysis_config": {
|
||||
"bucket_span": "5m",
|
||||
"latency": "0ms",
|
||||
"bucket_span": "10m",
|
||||
"detectors": [
|
||||
{
|
||||
"detector_description": "low_sum(events_per_min)",
|
||||
"function": "low_sum",
|
||||
"field_name": "events_per_min",
|
||||
"detector_description": "Sum of total",
|
||||
"function": "sum",
|
||||
"field_name": "total",
|
||||
"rules": [],
|
||||
"detector_index": 0
|
||||
}
|
||||
@ -133,13 +131,12 @@ When the job is created, you receive the following results:
|
||||
"model_memory_limit": "1024mb"
|
||||
},
|
||||
"data_description": {
|
||||
"time_field": "@timestamp",
|
||||
"time_field": "timestamp",
|
||||
"time_format": "epoch_ms"
|
||||
},
|
||||
"model_snapshot_retention_days": 1,
|
||||
"results_index_name": "shared"
|
||||
}
|
||||
----
|
||||
//CONSOLE
|
||||
//TESTRESPONSE[s/"job_version": "7.0.0-alpha1"/"job_version": $body.job_version/]
|
||||
//TESTRESPONSE[s/"create_time": 1502832478794/"create_time": $body.create_time/]
|
||||
// TESTRESPONSE[s/"job_version": "7.0.0-alpha1"/"job_version": $body.job_version/]
|
||||
// TESTRESPONSE[s/"create_time": 1517011406091/"create_time": $body.create_time/]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user