Merge branch 'master' into index-lifecycle
This commit is contained in:
Tal Levy
commit
86928e7d24
|
|
@ -115,7 +115,7 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P
|
|||
useFixture = true
|
||||
|
||||
} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
|
||||
throw new IllegalArgumentException("not all options specified to run against external S3 service")
|
||||
throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present")
|
||||
}
|
||||
|
||||
if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
|
||||
|
|
@ -126,7 +126,7 @@ if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3T
|
|||
s3TemporarySessionToken = 's3_integration_test_temporary_session_token'
|
||||
|
||||
} else if (!s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) {
|
||||
throw new IllegalArgumentException("not all options specified to run against external S3 service")
|
||||
throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present")
|
||||
}
|
||||
|
||||
final String minioVersion = 'RELEASE.2018-06-22T23-48-46Z'
|
||||
|
|
@ -381,13 +381,13 @@ integTestCluster {
|
|||
|
||||
integTestRunner.systemProperty 'tests.rest.blacklist', 'repository_s3/50_repository_ecs_credentials/*'
|
||||
|
||||
///
|
||||
RestIntegTestTask integTestECS = project.tasks.create('integTestECS', RestIntegTestTask.class) {
|
||||
if (useFixture) {
|
||||
RestIntegTestTask integTestECS = project.tasks.create('integTestECS', RestIntegTestTask.class) {
|
||||
description = "Runs tests using the ECS repository."
|
||||
}
|
||||
}
|
||||
|
||||
// The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks:
|
||||
project.afterEvaluate {
|
||||
project.afterEvaluate {
|
||||
ClusterConfiguration cluster = project.extensions.getByName('integTestECSCluster') as ClusterConfiguration
|
||||
cluster.dependsOn(project.s3Fixture)
|
||||
|
||||
|
|
@ -403,9 +403,9 @@ project.afterEvaluate {
|
|||
'repository_s3/30_repository_temporary_credentials/*',
|
||||
'repository_s3/40_repository_ec2_credentials/*'
|
||||
].join(",")
|
||||
}
|
||||
project.check.dependsOn(integTestECS)
|
||||
}
|
||||
project.check.dependsOn(integTestECS)
|
||||
///
|
||||
|
||||
thirdPartyAudit.excludes = [
|
||||
// classes are missing
|
||||
|
|
|
|||
|
|
@ -178,6 +178,8 @@ public class Version implements Comparable<Version>, ToXContentFragment {
|
|||
public static final Version V_6_3_3 = new Version(V_6_3_3_ID, org.apache.lucene.util.Version.LUCENE_7_3_1);
|
||||
public static final int V_6_4_0_ID = 6040099;
|
||||
public static final Version V_6_4_0 = new Version(V_6_4_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
|
||||
public static final int V_6_5_0_ID = 6050099;
|
||||
public static final Version V_6_5_0 = new Version(V_6_5_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
|
||||
public static final int V_7_0_0_alpha1_ID = 7000001;
|
||||
public static final Version V_7_0_0_alpha1 =
|
||||
new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
|
||||
|
|
@ -196,6 +198,8 @@ public class Version implements Comparable<Version>, ToXContentFragment {
|
|||
switch (id) {
|
||||
case V_7_0_0_alpha1_ID:
|
||||
return V_7_0_0_alpha1;
|
||||
case V_6_5_0_ID:
|
||||
return V_6_5_0;
|
||||
case V_6_4_0_ID:
|
||||
return V_6_4_0;
|
||||
case V_6_3_3_ID:
|
||||
|
|
|
|||
|
|
@ -310,6 +310,16 @@ setups['farequote_datafeed'] = setups['farequote_job'] + '''
|
|||
"indexes":"farequote"
|
||||
}
|
||||
'''
|
||||
setups['ml_filter_safe_domains'] = '''
|
||||
- do:
|
||||
xpack.ml.put_filter:
|
||||
filter_id: "safe_domains"
|
||||
body: >
|
||||
{
|
||||
"description": "A list of safe domains",
|
||||
"items": ["*.google.com", "wikipedia.org"]
|
||||
}
|
||||
'''
|
||||
setups['server_metrics_index'] = '''
|
||||
- do:
|
||||
indices.create:
|
||||
|
|
|
|||
|
|
@ -105,8 +105,8 @@ For all other aggregations, if the aggregation name doesn't match the field name
|
|||
there are limitations in the drill-down functionality within the {ml} page in
|
||||
{kib}.
|
||||
|
||||
{dfeeds} support complex nested aggregations, this example uses the `derivative`
|
||||
pipeline aggregation to find the 1st order derivative of the counter
|
||||
{dfeeds-cap} support complex nested aggregations, this example uses the `derivative`
|
||||
pipeline aggregation to find the first order derivative of the counter
|
||||
`system.network.out.bytes` for each value of the field `beat.name`.
|
||||
|
||||
[source,js]
|
||||
|
|
|
|||
|
|
@ -47,6 +47,15 @@ The main {ml} resources can be accessed with a variety of endpoints:
|
|||
* {ref}/ml-delete-calendar-job.html[DELETE /calendars/<calendar_id+++>+++/jobs/<job_id+++>+++]: Disassociate a job from a calendar
|
||||
* {ref}/ml-delete-calendar.html[DELETE /calendars/<calendar_id+++>+++]: Delete a calendar
|
||||
|
||||
[float]
|
||||
[[ml-api-filters]]
|
||||
=== /filters/
|
||||
|
||||
* {ref}/ml-put-filter.html[PUT /filters/<filter_id+++>+++]: Create a filter
|
||||
* {ref}/ml-update-filter.html[POST /filters/<filter_id+++>+++/_update]: Update a filter
|
||||
* {ref}/ml-get-filter.html[GET /filters/<filter_id+++>+++]: List filters
|
||||
* {ref}/ml-delete-filter.html[DELETE /filter/<filter_id+++>+++]: Delete a filter
|
||||
|
||||
[float]
|
||||
[[ml-api-datafeeds]]
|
||||
=== /datafeeds/
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ The scenarios in this section describe some best practices for generating useful
|
|||
* <<ml-configuring-categories>>
|
||||
* <<ml-configuring-pop>>
|
||||
* <<ml-configuring-transform>>
|
||||
* <<ml-configuring-detector-custom-rules>>
|
||||
|
||||
:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/ml/customurl.asciidoc
|
||||
include::customurl.asciidoc[]
|
||||
|
|
@ -49,3 +50,6 @@ include::populations.asciidoc[]
|
|||
|
||||
:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/ml/transforms.asciidoc
|
||||
include::transforms.asciidoc[]
|
||||
|
||||
:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/ml/detector-custom-rules.asciidoc
|
||||
include::detector-custom-rules.asciidoc[]
|
||||
|
|
@ -0,0 +1,230 @@
|
|||
[role="xpack"]
|
||||
[[ml-configuring-detector-custom-rules]]
|
||||
=== Customizing detectors with rules and filters
|
||||
|
||||
<<ml-rules,Rules and filters>> enable you to change the behavior of anomaly
|
||||
detectors based on domain-specific knowledge.
|
||||
|
||||
Rules describe _when_ a detector should take a certain _action_ instead
|
||||
of following its default behavior. To specify the _when_ a rule uses
|
||||
a `scope` and `conditions`. You can think of `scope` as the categorical
|
||||
specification of a rule, while `conditions` are the numerical part.
|
||||
A rule can have a scope, one or more conditions, or a combination of
|
||||
scope and conditions.
|
||||
|
||||
Let us see how those can be configured by examples.
|
||||
|
||||
==== Specifying rule scope
|
||||
|
||||
Let us assume we are configuring a job in order to DNS data exfiltration.
|
||||
Our data contain fields "subdomain" and "highest_registered_domain".
|
||||
We can use a detector that looks like `high_info_content(subdomain) over highest_registered_domain`.
|
||||
If we run such a job it is possible that we discover a lot of anomalies on
|
||||
frequently used domains that we have reasons to trust. As security analysts, we
|
||||
are not interested in such anomalies. Ideally, we could instruct the detector to
|
||||
skip results for domains that we consider safe. Using a rule with a scope allows
|
||||
us to achieve this.
|
||||
|
||||
First, we need to create a list with our safe domains. Those lists are called
|
||||
`filters` in {ml}. Filters can be shared across jobs.
|
||||
|
||||
We create our filter using the {ref}/ml-put-filter.html[put filter API]:
|
||||
|
||||
[source,js]
|
||||
----------------------------------
|
||||
PUT _xpack/ml/filters/safe_domains
|
||||
{
|
||||
"description": "Our list of safe domains",
|
||||
"items": ["safe.com", "trusted.com"]
|
||||
}
|
||||
----------------------------------
|
||||
// CONSOLE
|
||||
|
||||
Now, we can create our job specifying a scope that uses the filter for the
|
||||
`highest_registered_domain` field:
|
||||
|
||||
[source,js]
|
||||
----------------------------------
|
||||
PUT _xpack/ml/anomaly_detectors/dns_exfiltration_with_rule
|
||||
{
|
||||
"analysis_config" : {
|
||||
"bucket_span":"5m",
|
||||
"detectors" :[{
|
||||
"function":"high_info_content",
|
||||
"field_name": "subdomain",
|
||||
"over_field_name": "highest_registered_domain",
|
||||
"custom_rules": [{
|
||||
"actions": ["skip_result"],
|
||||
"scope": {
|
||||
"highest_registered_domain": {
|
||||
"filter_id": "safe_domains",
|
||||
"filter_type": "include"
|
||||
}
|
||||
}
|
||||
}]
|
||||
}]
|
||||
},
|
||||
"data_description" : {
|
||||
"time_field":"timestamp"
|
||||
}
|
||||
}
|
||||
----------------------------------
|
||||
// CONSOLE
|
||||
|
||||
As time advances and we see more data and more results, we might encounter new
|
||||
domains that we want to add in the filter. We can do that by using the
|
||||
{ref}/ml-update-filter.html[update filter API]:
|
||||
|
||||
[source,js]
|
||||
----------------------------------
|
||||
POST _xpack/ml/filters/safe_domains/_update
|
||||
{
|
||||
"add_items": ["another-safe.com"]
|
||||
}
|
||||
----------------------------------
|
||||
// CONSOLE
|
||||
// TEST[setup:ml_filter_safe_domains]
|
||||
|
||||
Note that we can provide scope for any of the partition/over/by fields.
|
||||
In the following example we scope multiple fields:
|
||||
|
||||
[source,js]
|
||||
----------------------------------
|
||||
PUT _xpack/ml/anomaly_detectors/scoping_multiple_fields
|
||||
{
|
||||
"analysis_config" : {
|
||||
"bucket_span":"5m",
|
||||
"detectors" :[{
|
||||
"function":"count",
|
||||
"partition_field_name": "my_partition",
|
||||
"over_field_name": "my_over",
|
||||
"by_field_name": "my_by",
|
||||
"custom_rules": [{
|
||||
"actions": ["skip_result"],
|
||||
"scope": {
|
||||
"my_partition": {
|
||||
"filter_id": "filter_1"
|
||||
},
|
||||
"my_over": {
|
||||
"filter_id": "filter_2"
|
||||
},
|
||||
"my_by": {
|
||||
"filter_id": "filter_3"
|
||||
}
|
||||
}
|
||||
}]
|
||||
}]
|
||||
},
|
||||
"data_description" : {
|
||||
"time_field":"timestamp"
|
||||
}
|
||||
}
|
||||
----------------------------------
|
||||
// CONSOLE
|
||||
|
||||
Such a detector will skip results when the values of all 3 scoped fields
|
||||
are included in the referenced filters.
|
||||
|
||||
==== Specifying rule conditions
|
||||
|
||||
Imagine a detector that looks for anomalies in CPU utilization.
|
||||
Given a machine that is idle for long enough, small movement in CPU could
|
||||
result in anomalous results where the `actual` value is quite small, for
|
||||
example, 0.02. Given our knowledge about how CPU utilization behaves we might
|
||||
determine that anomalies with such small actual values are not interesting for
|
||||
investigation.
|
||||
|
||||
Let us now configure a job with a rule that will skip results where CPU
|
||||
utilization is less than 0.20.
|
||||
|
||||
[source,js]
|
||||
----------------------------------
|
||||
PUT _xpack/ml/anomaly_detectors/cpu_with_rule
|
||||
{
|
||||
"analysis_config" : {
|
||||
"bucket_span":"5m",
|
||||
"detectors" :[{
|
||||
"function":"high_mean",
|
||||
"field_name": "cpu_utilization",
|
||||
"custom_rules": [{
|
||||
"actions": ["skip_result"],
|
||||
"conditions": [
|
||||
{
|
||||
"applies_to": "actual",
|
||||
"operator": "lt",
|
||||
"value": 0.20
|
||||
}
|
||||
]
|
||||
}]
|
||||
}]
|
||||
},
|
||||
"data_description" : {
|
||||
"time_field":"timestamp"
|
||||
}
|
||||
}
|
||||
----------------------------------
|
||||
// CONSOLE
|
||||
|
||||
When there are multiple conditions they are combined with a logical `and`.
|
||||
This is useful when we want the rule to apply to a range. We simply create
|
||||
a rule with two conditions, one for each end of the desired range.
|
||||
|
||||
Here is an example where a count detector will skip results when the count
|
||||
is greater than 30 and less than 50:
|
||||
|
||||
[source,js]
|
||||
----------------------------------
|
||||
PUT _xpack/ml/anomaly_detectors/rule_with_range
|
||||
{
|
||||
"analysis_config" : {
|
||||
"bucket_span":"5m",
|
||||
"detectors" :[{
|
||||
"function":"count",
|
||||
"custom_rules": [{
|
||||
"actions": ["skip_result"],
|
||||
"conditions": [
|
||||
{
|
||||
"applies_to": "actual",
|
||||
"operator": "gt",
|
||||
"value": 30
|
||||
},
|
||||
{
|
||||
"applies_to": "actual",
|
||||
"operator": "lt",
|
||||
"value": 50
|
||||
}
|
||||
]
|
||||
}]
|
||||
}]
|
||||
},
|
||||
"data_description" : {
|
||||
"time_field":"timestamp"
|
||||
}
|
||||
}
|
||||
----------------------------------
|
||||
// CONSOLE
|
||||
|
||||
==== Rules in the life-cycle of a job
|
||||
|
||||
Rules only affect results created after the rules were applied.
|
||||
Let us imagine that we have configured a job and it has been running
|
||||
for some time. After observing its results we decide that we can employ
|
||||
rules in order to get rid of some uninteresting results. We can use
|
||||
the update-job API to do so. However, the rule we added will only be in effect
|
||||
for any results created from the moment we added the rule onwards. Past results
|
||||
will remain unaffected.
|
||||
|
||||
==== Using rules VS filtering data
|
||||
|
||||
It might appear like using rules is just another way of filtering the data
|
||||
that feeds into a job. For example, a rule that skips results when the
|
||||
partition field value is in a filter sounds equivalent to having a query
|
||||
that filters out such documents. But it is not. There is a fundamental
|
||||
difference. When the data is filtered before reaching a job it is as if they
|
||||
never existed for the job. With rules, the data still reaches the job and
|
||||
affects its behavior (depending on the rule actions).
|
||||
|
||||
For example, a rule with the `skip_result` action means all data will still
|
||||
be modeled. On the other hand, a rule with the `skip_model_update` action means
|
||||
results will still be created even though the model will not be updated by
|
||||
data matched by a rule.
|
||||
|
|
@ -8,6 +8,8 @@ input data.
|
|||
The {xpackml} features include the following geographic function: `lat_long`.
|
||||
|
||||
NOTE: You cannot create forecasts for jobs that contain geographic functions.
|
||||
You also cannot add rules with conditions to detectors that use geographic
|
||||
functions.
|
||||
|
||||
[float]
|
||||
[[ml-lat-long]]
|
||||
|
|
|
|||
|
|
@ -15,6 +15,9 @@ The {xpackml} features include the following metric functions:
|
|||
* <<ml-metric-metric,`metric`>>
|
||||
* xref:ml-metric-varp[`varp`, `high_varp`, `low_varp`]
|
||||
|
||||
NOTE: You cannot add rules with conditions to detectors that use the `metric`
|
||||
function.
|
||||
|
||||
[float]
|
||||
[[ml-metric-min]]
|
||||
==== Min
|
||||
|
|
@ -221,7 +224,6 @@ mean `responsetime` for each application over time. It detects when the mean
|
|||
The `metric` function combines `min`, `max`, and `mean` functions. You can use
|
||||
it as a shorthand for a combined analysis. If you do not specify a function in
|
||||
a detector, this is the default function.
|
||||
//TBD: Is that default behavior still true?
|
||||
|
||||
High- and low-sided functions are not applicable. You cannot use this function
|
||||
when a `summary_count_field_name` is specified.
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ number of times (frequency) rare values occur.
|
|||
`exclude_frequent`.
|
||||
* You cannot create forecasts for jobs that contain `rare` or `freq_rare`
|
||||
functions.
|
||||
* You cannot add rules with conditions to detectors that use `rare` or
|
||||
`freq_rare` functions.
|
||||
* Shorter bucket spans (less than 1 hour, for example) are recommended when
|
||||
looking for rare events. The functions model whether something happens in a
|
||||
bucket at least once. With longer bucket spans, it is more likely that
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ job configuration options.
|
|||
* <<ml-calendar-resource,Calendars>>
|
||||
* <<ml-datafeed-resource,{dfeeds-cap}>>
|
||||
* <<ml-datafeed-counts,{dfeed-cap} counts>>
|
||||
* <<ml-filter-resource,Filters>>
|
||||
* <<ml-job-resource,Jobs>>
|
||||
* <<ml-jobstats,Job statistics>>
|
||||
* <<ml-snapshot-resource,Model snapshots>>
|
||||
|
|
@ -19,6 +20,8 @@ include::ml/calendarresource.asciidoc[]
|
|||
[role="xpack"]
|
||||
include::ml/datafeedresource.asciidoc[]
|
||||
[role="xpack"]
|
||||
include::ml/filterresource.asciidoc[]
|
||||
[role="xpack"]
|
||||
include::ml/jobresource.asciidoc[]
|
||||
[role="xpack"]
|
||||
include::ml/jobcounts.asciidoc[]
|
||||
|
|
|
|||
|
|
@ -15,6 +15,14 @@ machine learning APIs and in advanced job configuration options in Kibana.
|
|||
* <<ml-post-calendar-event,Add scheduled events to calendar>>, <<ml-delete-calendar-event,Delete scheduled events from calendar>>
|
||||
* <<ml-get-calendar,Get calendars>>, <<ml-get-calendar-event,Get scheduled events>>
|
||||
|
||||
[float]
|
||||
[[ml-api-filter-endpoint]]
|
||||
=== Filters
|
||||
|
||||
* <<ml-put-filter,Create filter>>, <<ml-delete-filter,Delete filter>>
|
||||
* <<ml-update-filter,Update filters>>
|
||||
* <<ml-get-filter,Get filters>>
|
||||
|
||||
[float]
|
||||
[[ml-api-datafeed-endpoint]]
|
||||
=== {dfeeds-cap}
|
||||
|
|
@ -69,11 +77,13 @@ include::ml/close-job.asciidoc[]
|
|||
//CREATE
|
||||
include::ml/put-calendar.asciidoc[]
|
||||
include::ml/put-datafeed.asciidoc[]
|
||||
include::ml/put-filter.asciidoc[]
|
||||
include::ml/put-job.asciidoc[]
|
||||
//DELETE
|
||||
include::ml/delete-calendar.asciidoc[]
|
||||
include::ml/delete-datafeed.asciidoc[]
|
||||
include::ml/delete-calendar-event.asciidoc[]
|
||||
include::ml/delete-filter.asciidoc[]
|
||||
include::ml/delete-job.asciidoc[]
|
||||
include::ml/delete-calendar-job.asciidoc[]
|
||||
include::ml/delete-snapshot.asciidoc[]
|
||||
|
|
@ -93,6 +103,7 @@ include::ml/get-job.asciidoc[]
|
|||
include::ml/get-job-stats.asciidoc[]
|
||||
include::ml/get-snapshot.asciidoc[]
|
||||
include::ml/get-calendar-event.asciidoc[]
|
||||
include::ml/get-filter.asciidoc[]
|
||||
include::ml/get-record.asciidoc[]
|
||||
//OPEN
|
||||
include::ml/open-job.asciidoc[]
|
||||
|
|
@ -107,6 +118,7 @@ include::ml/start-datafeed.asciidoc[]
|
|||
include::ml/stop-datafeed.asciidoc[]
|
||||
//UPDATE
|
||||
include::ml/update-datafeed.asciidoc[]
|
||||
include::ml/update-filter.asciidoc[]
|
||||
include::ml/update-job.asciidoc[]
|
||||
include::ml/update-snapshot.asciidoc[]
|
||||
//VALIDATE
|
||||
|
|
|
|||
|
|
@ -0,0 +1,53 @@
|
|||
[role="xpack"]
|
||||
[[ml-delete-filter]]
|
||||
=== Delete Filter API
|
||||
++++
|
||||
<titleabbrev>Delete Filter</titleabbrev>
|
||||
++++
|
||||
|
||||
Deletes a filter.
|
||||
|
||||
|
||||
==== Request
|
||||
|
||||
`DELETE _xpack/ml/filters/<filter_id>`
|
||||
|
||||
|
||||
==== Description
|
||||
|
||||
This API deletes a {stack-ov}/ml-rules.html[filter].
|
||||
If a {ml} job references the filter, you cannot delete the filter. You must
|
||||
update or delete the job before you can delete the filter.
|
||||
|
||||
|
||||
==== Path Parameters
|
||||
|
||||
`filter_id` (required)::
|
||||
(string) Identifier for the filter.
|
||||
|
||||
|
||||
==== Authorization
|
||||
|
||||
You must have `manage_ml`, or `manage` cluster privileges to use this API.
|
||||
For more information, see {xpack-ref}/security-privileges.html[Security Privileges].
|
||||
|
||||
|
||||
==== Examples
|
||||
|
||||
The following example deletes the `safe_domains` filter:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
DELETE _xpack/ml/filters/safe_domains
|
||||
--------------------------------------------------
|
||||
// CONSOLE
|
||||
// TEST[setup:ml_filter_safe_domains]
|
||||
|
||||
When the filter is deleted, you receive the following results:
|
||||
[source,js]
|
||||
----
|
||||
{
|
||||
"acknowledged": true
|
||||
}
|
||||
----
|
||||
//TESTRESPONSE
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
[role="xpack"]
|
||||
[[ml-filter-resource]]
|
||||
=== Filter Resources
|
||||
|
||||
A filter resource has the following properties:
|
||||
|
||||
`filter_id`::
|
||||
(string) A string that uniquely identifies the filter.
|
||||
|
||||
`description`::
|
||||
(array) A description of the filter.
|
||||
|
||||
`items`::
|
||||
(array of strings) An array of strings which is the filter item list.
|
||||
|
||||
For more information, see {stack-ov}/ml-rules.html[Machine learning rules and filters].
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
[role="xpack"]
|
||||
[[ml-get-filter]]
|
||||
=== Get Filters API
|
||||
++++
|
||||
<titleabbrev>Get Filters</titleabbrev>
|
||||
++++
|
||||
|
||||
Retrieves filters.
|
||||
|
||||
|
||||
==== Request
|
||||
|
||||
`GET _xpack/ml/filters/<filter_id>` +
|
||||
|
||||
`GET _xpack/ml/filters/`
|
||||
|
||||
|
||||
===== Description
|
||||
|
||||
You can get a single filter or all filters. For more information, see
|
||||
{stack-ov}/ml-rules.html[Machine learning rules and filters].
|
||||
|
||||
|
||||
==== Path Parameters
|
||||
|
||||
`filter_id`::
|
||||
(string) Identifier for the filter.
|
||||
|
||||
|
||||
==== Request Body
|
||||
|
||||
`from`:::
|
||||
(integer) Skips the specified number of filters.
|
||||
|
||||
`size`:::
|
||||
(integer) Specifies the maximum number of filters to obtain.
|
||||
|
||||
|
||||
==== Results
|
||||
|
||||
The API returns the following information:
|
||||
|
||||
`filters`::
|
||||
(array) An array of filter resources.
|
||||
For more information, see <<ml-filter-resource>>.
|
||||
|
||||
|
||||
==== Authorization
|
||||
|
||||
You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster
|
||||
privileges to use this API. For more information, see
|
||||
{xpack-ref}/security-privileges.html[Security Privileges].
|
||||
|
||||
|
||||
==== Examples
|
||||
|
||||
The following example gets configuration information for the `safe_domains`
|
||||
filter:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
GET _xpack/ml/filters/safe_domains
|
||||
--------------------------------------------------
|
||||
// CONSOLE
|
||||
// TEST[setup:ml_filter_safe_domains]
|
||||
|
||||
The API returns the following results:
|
||||
[source,js]
|
||||
----
|
||||
{
|
||||
"count": 1,
|
||||
"filters": [
|
||||
{
|
||||
"filter_id": "safe_domains",
|
||||
"description": "A list of safe domains",
|
||||
"items": [
|
||||
"*.google.com",
|
||||
"wikipedia.org"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
----
|
||||
//TESTRESPONSE
|
||||
|
|
@ -106,7 +106,8 @@ An analysis configuration object has the following properties:
|
|||
|
||||
`bucket_span`::
|
||||
(time units) The size of the interval that the analysis is aggregated into,
|
||||
typically between `5m` and `1h`. The default value is `5m`.
|
||||
typically between `5m` and `1h`. The default value is `5m`. For more
|
||||
information about time units, see <<time-units,Common options>>.
|
||||
|
||||
`categorization_field_name`::
|
||||
(string) If this property is specified, the values of the specified field will
|
||||
|
|
@ -160,8 +161,7 @@ no analysis can occur and an error is returned.
|
|||
(time units) The size of the window in which to expect data that is out of
|
||||
time order. The default value is 0 (no latency). If you specify a non-zero
|
||||
value, it must be greater than or equal to one second. For more information
|
||||
about time units, see
|
||||
{ref}/common-options.html#time-units[Time Units].
|
||||
about time units, see <<time-units,Common options>>.
|
||||
+
|
||||
--
|
||||
NOTE: Latency is only applicable when you send data by using
|
||||
|
|
@ -262,7 +262,12 @@ NOTE: The `field_name` cannot contain double quotes or backslashes.
|
|||
|
||||
`use_null`::
|
||||
(boolean) Defines whether a new series is used as the null series
|
||||
when there is no value for the by or partition fields. The default value is `false`. +
|
||||
when there is no value for the by or partition fields. The default value is `false`.
|
||||
|
||||
`custom_rules`::
|
||||
(array) An array of rule objects, which enable customizing how the detector works.
|
||||
For example, a rule may dictate to the detector conditions under which results should be skipped.
|
||||
For more information see <<ml-detector-custom-rule,detector custom rule objects>>. +
|
||||
+
|
||||
--
|
||||
IMPORTANT: Field names are case sensitive, for example a field named 'Bytes'
|
||||
|
|
@ -270,9 +275,9 @@ is different from one named 'bytes'.
|
|||
|
||||
--
|
||||
|
||||
After you create a job, the only property you can change in the detector
|
||||
configuration object is the `detector_description`; all other properties are
|
||||
informational.
|
||||
After you create a job, the only properties you can change in the detector
|
||||
configuration object are the `detector_description` and the `custom_rules`;
|
||||
all other properties are informational.
|
||||
|
||||
[float]
|
||||
[[ml-datadescription]]
|
||||
|
|
@ -408,6 +413,64 @@ the categorization field value came from.
|
|||
For more information, see
|
||||
{xpack-ref}/ml-configuring-categories.html[Categorizing Log Messages].
|
||||
|
||||
[float]
|
||||
[[ml-detector-custom-rule]]
|
||||
==== Detector Custom Rule
|
||||
|
||||
{stack-ov}/ml-rules.html[Custom rules] enable you to customize the way detectors
|
||||
operate.
|
||||
|
||||
A rule has the following properties:
|
||||
|
||||
`actions`::
|
||||
(array) The set of actions to be triggered when the rule applies.
|
||||
If more than one action is specified the effects of all actions are combined.
|
||||
The available actions include: +
|
||||
`skip_result`::: The result will not be created. This is the default value.
|
||||
Unless you also specify `skip_model_update`, the model will be updated as
|
||||
usual with the corresponding series value.
|
||||
`skip_model_update`::: The value for that series will not be used to update
|
||||
the model. Unless you also specify `skip_result`, the results will be created
|
||||
as usual. This action is suitable when certain values are expected to be
|
||||
consistently anomalous and they affect the model in a way that negatively
|
||||
impacts the rest of the results.
|
||||
`scope`::
|
||||
(object) An optional scope of series where the rule applies. By default the scope
|
||||
includes all series. Scoping is allowed for any of the partition/by/over fields.
|
||||
To add a scope for a field add the field name as a key in the scope object and
|
||||
set its value to an object with properties:
|
||||
`filter_id`::
|
||||
(string) The id of the <<ml-filter-resource,filter>> to be used.
|
||||
`filter_type`::
|
||||
(string) Either `include` (the rule applies for values in the filter)
|
||||
or `exclude` (the rule applies for values not in the filter). Defaults
|
||||
to `include`.
|
||||
|
||||
`conditions`::
|
||||
(array) An optional array of numeric conditions when the rule applies.
|
||||
Multiple conditions are combined together with a logical `AND`.
|
||||
+
|
||||
--
|
||||
NOTE: If your detector uses `lat_long`, `metric`, `rare`, or `freq_rare`
|
||||
functions, you cannot specify `conditions` for your rule.
|
||||
|
||||
|
||||
A condition has the following properties:
|
||||
|
||||
`applies_to`:::
|
||||
(string) Specifies the result property to which the condition applies.
|
||||
The available options are `actual`, `typical`, `diff_from_typical`, `time`.
|
||||
`operator`:::
|
||||
(string) Specifies the condition operator. The available options are
|
||||
`gt` (greater than), `gte` (greater than or equals), `lt` (less than) and `lte` (less than or equals).
|
||||
`value`:::
|
||||
(double) The value that is compared against the `applied_to` field using the `operator`.
|
||||
--
|
||||
|
||||
A rule is required to either have a non-empty scope or at least one condition.
|
||||
For more examples see
|
||||
{stack-ov}/ml-configuring-detector-custom-rules.html[Configuring Detector Custom Rules].
|
||||
|
||||
[float]
|
||||
[[ml-apilimits]]
|
||||
==== Analysis Limits
|
||||
|
|
@ -448,8 +511,7 @@ Specifying a string is recommended for clarity. If you specify a byte size unit
|
|||
of `b` or `kb` and the number does not equate to a discrete number of megabytes,
|
||||
it is rounded down to the closest MiB. The minimum valid value is 1 MiB. If you
|
||||
specify a value less than 1 MiB, an error occurs. For more information about
|
||||
supported byte size units, see
|
||||
{ref}/common-options.html#byte-units[Byte size units].
|
||||
supported byte size units, see <<byte-units,Common options>>.
|
||||
|
||||
If your `elasticsearch.yml` file contains an `xpack.ml.max_model_memory_limit`
|
||||
setting, an error occurs when you try to create jobs that have
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
[role="xpack"]
|
||||
[[ml-put-filter]]
|
||||
=== Create Filter API
|
||||
++++
|
||||
<titleabbrev>Create Filter</titleabbrev>
|
||||
++++
|
||||
|
||||
Instantiates a filter.
|
||||
|
||||
==== Request
|
||||
|
||||
`PUT _xpack/ml/filters/<filter_id>`
|
||||
|
||||
===== Description
|
||||
|
||||
A {stack-ov}/ml-rules.html[filter] contains a list of strings.
|
||||
It can be used by one or more jobs. Specifically, filters are referenced in
|
||||
the `custom_rules` property of <<ml-detectorconfig,detector configuration objects>>.
|
||||
|
||||
==== Path Parameters
|
||||
|
||||
`filter_id` (required)::
|
||||
(string) Identifier for the filter.
|
||||
|
||||
|
||||
==== Request Body
|
||||
|
||||
`description`::
|
||||
(string) A description of the filter.
|
||||
|
||||
`items`::
|
||||
(array of strings) The items of the filter.
|
||||
A wildcard `*` can be used at the beginning
|
||||
or the end of an item. Up to 10000 items
|
||||
are allowed in each filter.
|
||||
|
||||
|
||||
==== Authorization
|
||||
|
||||
You must have `manage_ml`, or `manage` cluster privileges to use this API.
|
||||
For more information, see
|
||||
{xpack-ref}/security-privileges.html[Security Privileges].
|
||||
|
||||
|
||||
==== Examples
|
||||
|
||||
The following example creates the `safe_domains` filter:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
PUT _xpack/ml/filters/safe_domains
|
||||
{
|
||||
"description": "A list of safe domains",
|
||||
"items": ["*.google.com", "wikipedia.org"]
|
||||
}
|
||||
--------------------------------------------------
|
||||
// CONSOLE
|
||||
|
||||
When the filter is created, you receive the following response:
|
||||
[source,js]
|
||||
----
|
||||
{
|
||||
"filter_id": "safe_domains",
|
||||
"description": "A list of safe domains",
|
||||
"items": ["*.google.com", "wikipedia.org"]
|
||||
}
|
||||
----
|
||||
//TESTRESPONSE
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
[role="xpack"]
|
||||
[[ml-update-filter]]
|
||||
=== Update Filter API
|
||||
++++
|
||||
<titleabbrev>Update Filter</titleabbrev>
|
||||
++++
|
||||
|
||||
Updates the description of a filter, adds items, or removes items.
|
||||
|
||||
==== Request
|
||||
|
||||
`POST _xpack/ml/filters/<filter_id>/_update`
|
||||
|
||||
//==== Description
|
||||
|
||||
==== Path Parameters
|
||||
|
||||
`filter_id` (required)::
|
||||
(string) Identifier for the filter.
|
||||
|
||||
|
||||
==== Request Body
|
||||
|
||||
`description`::
|
||||
(string) A description for the filter. See <<ml-filter-resource>>.
|
||||
|
||||
`add_items`::
|
||||
(array of strings) The items to add to the filter.
|
||||
|
||||
`remove_items`::
|
||||
(array of strings) The items to remove from the filter.
|
||||
|
||||
|
||||
==== Authorization
|
||||
|
||||
You must have `manage_ml`, or `manage` cluster privileges to use this API.
|
||||
For more information, see
|
||||
{xpack-ref}/security-privileges.html[Security Privileges].
|
||||
|
||||
|
||||
==== Examples
|
||||
|
||||
You can change the description, add and remove items to the `safe_domains` filter as follows:
|
||||
|
||||
[source,js]
|
||||
--------------------------------------------------
|
||||
POST _xpack/ml/filters/safe_domains/_update
|
||||
{
|
||||
"description": "Updated list of domains",
|
||||
"add_items": ["*.myorg.com"],
|
||||
"remove_items": ["wikipedia.org"]
|
||||
}
|
||||
--------------------------------------------------
|
||||
// CONSOLE
|
||||
// TEST[setup:ml_filter_safe_domains]
|
||||
|
||||
The API returns the following results:
|
||||
|
||||
[source,js]
|
||||
----
|
||||
{
|
||||
"filter_id": "safe_domains",
|
||||
"description": "Updated list of domains",
|
||||
"items": ["*.google.com", "*.myorg.com"]
|
||||
}
|
||||
----
|
||||
//TESTRESPONSE
|
||||
|
|
@ -35,6 +35,8 @@ each periodic persistence of the model. See <<ml-job-resource>>. | Yes
|
|||
|
||||
|`description` |A description of the job. See <<ml-job-resource>>. | No
|
||||
|
||||
|`detectors` |An array of <<ml-detector-update, detector update objects>>. | No
|
||||
|
||||
|`groups` |A list of job groups. See <<ml-job-resource>>. | No
|
||||
|
||||
|`model_plot_config`: `enabled` |If true, enables calculation and storage of the
|
||||
|
|
@ -58,12 +60,6 @@ if the job is open when you make the update, you must stop the data feed, close
|
|||
the job, then restart the data feed and open the job for the changes to take
|
||||
effect.
|
||||
|
||||
//|`analysis_config`: `detectors`: `detector_index` | A unique identifier of the
|
||||
//detector. Matches the order of detectors returned by
|
||||
//<<ml-get-job,GET job>>, starting from 0. | No
|
||||
//|`analysis_config`: `detectors`: `detector_description` |A description of the
|
||||
//detector. See <<ml-analysisconfig>>. | No
|
||||
|
||||
[NOTE]
|
||||
--
|
||||
* You can update the `analysis_limits` only while the job is closed.
|
||||
|
|
@ -73,6 +69,21 @@ of `hard_limit`, this means that it was unable to process some data. You might
|
|||
want to re-run this job with an increased `model_memory_limit`.
|
||||
--
|
||||
|
||||
[[ml-detector-update]]
|
||||
==== Detector Update Objects
|
||||
|
||||
A detector update object has the following properties:
|
||||
|
||||
`detector_index`::
|
||||
(integer) The identifier of the detector to update.
|
||||
|
||||
`description`::
|
||||
(string) The new description for the detector.
|
||||
|
||||
`custom_rules`::
|
||||
(array) The new list of <<ml-detector-custom-rule, rules>> for the detector.
|
||||
|
||||
No other detector property can be updated.
|
||||
|
||||
==== Authorization
|
||||
|
||||
|
|
|
|||
|
|
@ -80,6 +80,9 @@
|
|||
},
|
||||
"rule": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"opaque_id": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -508,10 +508,12 @@ public class RollupResponseTranslationTests extends AggregatorTestCase {
|
|||
BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
|
||||
ScriptService scriptService = mock(ScriptService.class);
|
||||
InternalAggregation.ReduceContext reduceContext = new InternalAggregation.ReduceContext(bigArrays, scriptService, true);
|
||||
Exception e = expectThrows(RuntimeException.class,
|
||||
ClassCastException e = expectThrows(ClassCastException.class,
|
||||
() -> RollupResponseTranslator.combineResponses(msearch, reduceContext));
|
||||
assertThat(e.getMessage(), equalTo("org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds " +
|
||||
"cannot be cast to org.elasticsearch.search.aggregations.InternalMultiBucketAggregation"));
|
||||
assertThat(e.getMessage(),
|
||||
containsString("org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds"));
|
||||
assertThat(e.getMessage(),
|
||||
containsString("org.elasticsearch.search.aggregations.InternalMultiBucketAggregation"));
|
||||
}
|
||||
|
||||
public void testDateHisto() throws IOException {
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ import org.elasticsearch.gateway.GatewayService;
|
|||
import org.elasticsearch.node.Node;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.rest.RestRequest;
|
||||
import org.elasticsearch.tasks.Task;
|
||||
import org.elasticsearch.threadpool.ThreadPool;
|
||||
import org.elasticsearch.transport.TransportMessage;
|
||||
import org.elasticsearch.xpack.core.XPackClientPlugin;
|
||||
|
|
@ -882,6 +883,12 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail, Cl
|
|||
builder.field(Field.NODE_HOST_ADDRESS, nodeHostAddress);
|
||||
builder.field(Field.LAYER, layer);
|
||||
builder.field(Field.TYPE, type);
|
||||
|
||||
String opaqueId = threadPool.getThreadContext().getHeader(Task.X_OPAQUE_ID);
|
||||
if (opaqueId != null) {
|
||||
builder.field("opaque_id", opaqueId);
|
||||
}
|
||||
|
||||
return builder;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@ import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplat
|
|||
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.client.Client;
|
||||
import org.elasticsearch.client.Request;
|
||||
import org.elasticsearch.client.RequestOptions;
|
||||
import org.elasticsearch.client.Response;
|
||||
import org.elasticsearch.cluster.ClusterState;
|
||||
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
|
||||
|
|
@ -18,8 +20,10 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
|
|||
import org.elasticsearch.common.network.NetworkModule;
|
||||
import org.elasticsearch.common.settings.SecureString;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.query.QueryBuilder;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.tasks.Task;
|
||||
import org.elasticsearch.test.ESIntegTestCase;
|
||||
import org.elasticsearch.test.TestCluster;
|
||||
import org.elasticsearch.xpack.core.XPackClientPlugin;
|
||||
|
|
@ -112,8 +116,53 @@ public class IndexAuditIT extends ESIntegTestCase {
|
|||
UsernamePasswordToken.basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray()))));
|
||||
assertThat(response.getStatusLine().getStatusCode(), is(200));
|
||||
final AtomicReference<ClusterState> lastClusterState = new AtomicReference<>();
|
||||
final boolean found = awaitSecurityAuditIndex(lastClusterState, QueryBuilders.matchQuery("principal", USER));
|
||||
|
||||
assertTrue("Did not find security audit index. Current cluster state:\n" + lastClusterState.get().toString(), found);
|
||||
|
||||
SearchResponse searchResponse = client().prepareSearch(".security_audit_log*").setQuery(
|
||||
QueryBuilders.matchQuery("principal", USER)).get();
|
||||
assertThat(searchResponse.getHits().getHits().length, greaterThan(0));
|
||||
assertThat(searchResponse.getHits().getAt(0).getSourceAsMap().get("principal"), is(USER));
|
||||
}
|
||||
|
||||
public void testAuditTrailTemplateIsRecreatedAfterDelete() throws Exception {
|
||||
// this is already "tested" by the test framework since we wipe the templates before and after,
|
||||
// but lets be explicit about the behavior
|
||||
awaitIndexTemplateCreation();
|
||||
|
||||
// delete the template
|
||||
DeleteIndexTemplateResponse deleteResponse = client().admin().indices()
|
||||
.prepareDeleteTemplate(IndexAuditTrail.INDEX_TEMPLATE_NAME).execute().actionGet();
|
||||
assertThat(deleteResponse.isAcknowledged(), is(true));
|
||||
awaitIndexTemplateCreation();
|
||||
}
|
||||
|
||||
public void testOpaqueIdWorking() throws Exception {
|
||||
Request request = new Request("GET", "/");
|
||||
RequestOptions.Builder options = request.getOptions().toBuilder();
|
||||
options.addHeader(Task.X_OPAQUE_ID, "foo");
|
||||
options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER,
|
||||
UsernamePasswordToken.basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray())));
|
||||
request.setOptions(options);
|
||||
Response response = getRestClient().performRequest(request);
|
||||
assertThat(response.getStatusLine().getStatusCode(), is(200));
|
||||
final AtomicReference<ClusterState> lastClusterState = new AtomicReference<>();
|
||||
final boolean found = awaitSecurityAuditIndex(lastClusterState, QueryBuilders.matchQuery("opaque_id", "foo"));
|
||||
|
||||
assertTrue("Did not find security audit index. Current cluster state:\n" + lastClusterState.get().toString(), found);
|
||||
|
||||
SearchResponse searchResponse = client().prepareSearch(".security_audit_log*").setQuery(
|
||||
QueryBuilders.matchQuery("opaque_id", "foo")).get();
|
||||
assertThat(searchResponse.getHits().getHits().length, greaterThan(0));
|
||||
|
||||
assertThat(searchResponse.getHits().getAt(0).getSourceAsMap().get("opaque_id"), is("foo"));
|
||||
}
|
||||
|
||||
private boolean awaitSecurityAuditIndex(AtomicReference<ClusterState> lastClusterState,
|
||||
QueryBuilder query) throws InterruptedException {
|
||||
final AtomicBoolean indexExists = new AtomicBoolean(false);
|
||||
final boolean found = awaitBusy(() -> {
|
||||
return awaitBusy(() -> {
|
||||
if (indexExists.get() == false) {
|
||||
ClusterState state = client().admin().cluster().prepareState().get().getState();
|
||||
lastClusterState.set(state);
|
||||
|
|
@ -138,28 +187,9 @@ public class IndexAuditIT extends ESIntegTestCase {
|
|||
logger.info("refreshing audit indices");
|
||||
client().admin().indices().prepareRefresh(".security_audit_log*").get();
|
||||
logger.info("refreshed audit indices");
|
||||
return client().prepareSearch(".security_audit_log*").setQuery(QueryBuilders.matchQuery("principal", USER))
|
||||
return client().prepareSearch(".security_audit_log*").setQuery(query)
|
||||
.get().getHits().getTotalHits() > 0;
|
||||
}, 60L, TimeUnit.SECONDS);
|
||||
|
||||
assertTrue("Did not find security audit index. Current cluster state:\n" + lastClusterState.get().toString(), found);
|
||||
|
||||
SearchResponse searchResponse = client().prepareSearch(".security_audit_log*").setQuery(
|
||||
QueryBuilders.matchQuery("principal", USER)).get();
|
||||
assertThat(searchResponse.getHits().getHits().length, greaterThan(0));
|
||||
assertThat(searchResponse.getHits().getAt(0).getSourceAsMap().get("principal"), is(USER));
|
||||
}
|
||||
|
||||
public void testAuditTrailTemplateIsRecreatedAfterDelete() throws Exception {
|
||||
// this is already "tested" by the test framework since we wipe the templates before and after,
|
||||
// but lets be explicit about the behavior
|
||||
awaitIndexTemplateCreation();
|
||||
|
||||
// delete the template
|
||||
DeleteIndexTemplateResponse deleteResponse = client().admin().indices()
|
||||
.prepareDeleteTemplate(IndexAuditTrail.INDEX_TEMPLATE_NAME).execute().actionGet();
|
||||
assertThat(deleteResponse.isAcknowledged(), is(true));
|
||||
awaitIndexTemplateCreation();
|
||||
}
|
||||
|
||||
private void awaitIndexTemplateCreation() throws InterruptedException {
|
||||
|
|
|
|||
Loading…
Reference in New Issue