From a2547168d27f0aada27d824f0d7a596a1a2c4fa1 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Wed, 25 Jul 2018 16:34:30 +0200 Subject: [PATCH 1/7] fixes broken build for third-party-tests (#32353) fixes broken build repository-s3 for third-party-tests --- plugins/repository-s3/build.gradle | 44 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 13119913672..6001ed57065 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -115,7 +115,7 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P useFixture = true } else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) { - throw new IllegalArgumentException("not all options specified to run against external S3 service") + throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present") } if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) { @@ -126,7 +126,7 @@ if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3T s3TemporarySessionToken = 's3_integration_test_temporary_session_token' } else if (!s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) { - throw new IllegalArgumentException("not all options specified to run against external S3 service") + throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present") } final String minioVersion = 'RELEASE.2018-06-22T23-48-46Z' @@ -381,31 +381,31 @@ integTestCluster { integTestRunner.systemProperty 'tests.rest.blacklist', 'repository_s3/50_repository_ecs_credentials/*' -/// -RestIntegTestTask integTestECS = project.tasks.create('integTestECS', RestIntegTestTask.class) { - description = "Runs tests using the ECS repository." -} +if (useFixture) { + RestIntegTestTask integTestECS = project.tasks.create('integTestECS', RestIntegTestTask.class) { + description = "Runs tests using the ECS repository." + } // The following closure must execute before the afterEvaluate block in the constructor of the following integrationTest tasks: -project.afterEvaluate { - ClusterConfiguration cluster = project.extensions.getByName('integTestECSCluster') as ClusterConfiguration - cluster.dependsOn(project.s3Fixture) + project.afterEvaluate { + ClusterConfiguration cluster = project.extensions.getByName('integTestECSCluster') as ClusterConfiguration + cluster.dependsOn(project.s3Fixture) - cluster.setting 's3.client.integration_test_ecs.endpoint', "http://${-> s3Fixture.addressAndPort}" + cluster.setting 's3.client.integration_test_ecs.endpoint', "http://${-> s3Fixture.addressAndPort}" - Task integTestECSTask = project.tasks.getByName('integTestECS') - integTestECSTask.clusterConfig.plugin(project.path) - integTestECSTask.clusterConfig.environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', - "http://${-> s3Fixture.addressAndPort}/ecs_credentials_endpoint" - integTestECSRunner.systemProperty 'tests.rest.blacklist', [ - 'repository_s3/10_basic/*', - 'repository_s3/20_repository_permanent_credentials/*', - 'repository_s3/30_repository_temporary_credentials/*', - 'repository_s3/40_repository_ec2_credentials/*' - ].join(",") + Task integTestECSTask = project.tasks.getByName('integTestECS') + integTestECSTask.clusterConfig.plugin(project.path) + integTestECSTask.clusterConfig.environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', + "http://${-> s3Fixture.addressAndPort}/ecs_credentials_endpoint" + integTestECSRunner.systemProperty 'tests.rest.blacklist', [ + 'repository_s3/10_basic/*', + 'repository_s3/20_repository_permanent_credentials/*', + 'repository_s3/30_repository_temporary_credentials/*', + 'repository_s3/40_repository_ec2_credentials/*' + ].join(",") + } + project.check.dependsOn(integTestECS) } -project.check.dependsOn(integTestECS) -/// thirdPartyAudit.excludes = [ // classes are missing From 9efd1407d5875c34e9f661fd9743c8bd78155878 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Wed, 25 Jul 2018 15:57:07 +0100 Subject: [PATCH 2/7] Add 6.5.0 version to master This commit adds the 6.5.0 version constant to the master branch. --- server/src/main/java/org/elasticsearch/Version.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index da0e3a3b3b1..1fefbad0d74 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -178,6 +178,8 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_6_3_3 = new Version(V_6_3_3_ID, org.apache.lucene.util.Version.LUCENE_7_3_1); public static final int V_6_4_0_ID = 6040099; public static final Version V_6_4_0 = new Version(V_6_4_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0); + public static final int V_6_5_0_ID = 6050099; + public static final Version V_6_5_0 = new Version(V_6_5_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0); public static final int V_7_0_0_alpha1_ID = 7000001; public static final Version V_7_0_0_alpha1 = new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_4_0); @@ -196,6 +198,8 @@ public class Version implements Comparable, ToXContentFragment { switch (id) { case V_7_0_0_alpha1_ID: return V_7_0_0_alpha1; + case V_6_5_0_ID: + return V_6_5_0; case V_6_4_0_ID: return V_6_4_0; case V_6_3_3_ID: From e12e2e0cdda7745fcfb1570a706d1c8ede21d6e6 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 25 Jul 2018 08:07:09 -0700 Subject: [PATCH 3/7] Add opaque_id to index audit logging (#32260) Logs opaque_id if it is available with all audit log messages using index-based audit log. Closes #31521 --- .../main/resources/security_audit_log.json | 3 + .../security/audit/index/IndexAuditTrail.java | 7 ++ .../xpack/security/audit/IndexAuditIT.java | 74 +++++++++++++------ 3 files changed, 62 insertions(+), 22 deletions(-) diff --git a/x-pack/plugin/core/src/main/resources/security_audit_log.json b/x-pack/plugin/core/src/main/resources/security_audit_log.json index f5decbb4019..75c25ff53e2 100644 --- a/x-pack/plugin/core/src/main/resources/security_audit_log.json +++ b/x-pack/plugin/core/src/main/resources/security_audit_log.json @@ -80,6 +80,9 @@ }, "rule": { "type": "keyword" + }, + "opaque_id": { + "type": "keyword" } } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java index 1991c2685f2..d8b4b4e4bc1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/index/IndexAuditTrail.java @@ -47,6 +47,7 @@ import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportMessage; import org.elasticsearch.xpack.core.XPackClientPlugin; @@ -882,6 +883,12 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail, Cl builder.field(Field.NODE_HOST_ADDRESS, nodeHostAddress); builder.field(Field.LAYER, layer); builder.field(Field.TYPE, type); + + String opaqueId = threadPool.getThreadContext().getHeader(Task.X_OPAQUE_ID); + if (opaqueId != null) { + builder.field("opaque_id", opaqueId); + } + return builder; } diff --git a/x-pack/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java b/x-pack/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java index 3467316c24f..a8155b785b3 100644 --- a/x-pack/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java +++ b/x-pack/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java @@ -11,6 +11,8 @@ import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplat import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; @@ -18,8 +20,10 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.TestCluster; import org.elasticsearch.xpack.core.XPackClientPlugin; @@ -112,8 +116,53 @@ public class IndexAuditIT extends ESIntegTestCase { UsernamePasswordToken.basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray())))); assertThat(response.getStatusLine().getStatusCode(), is(200)); final AtomicReference lastClusterState = new AtomicReference<>(); + final boolean found = awaitSecurityAuditIndex(lastClusterState, QueryBuilders.matchQuery("principal", USER)); + + assertTrue("Did not find security audit index. Current cluster state:\n" + lastClusterState.get().toString(), found); + + SearchResponse searchResponse = client().prepareSearch(".security_audit_log*").setQuery( + QueryBuilders.matchQuery("principal", USER)).get(); + assertThat(searchResponse.getHits().getHits().length, greaterThan(0)); + assertThat(searchResponse.getHits().getAt(0).getSourceAsMap().get("principal"), is(USER)); + } + + public void testAuditTrailTemplateIsRecreatedAfterDelete() throws Exception { + // this is already "tested" by the test framework since we wipe the templates before and after, + // but lets be explicit about the behavior + awaitIndexTemplateCreation(); + + // delete the template + DeleteIndexTemplateResponse deleteResponse = client().admin().indices() + .prepareDeleteTemplate(IndexAuditTrail.INDEX_TEMPLATE_NAME).execute().actionGet(); + assertThat(deleteResponse.isAcknowledged(), is(true)); + awaitIndexTemplateCreation(); + } + + public void testOpaqueIdWorking() throws Exception { + Request request = new Request("GET", "/"); + RequestOptions.Builder options = request.getOptions().toBuilder(); + options.addHeader(Task.X_OPAQUE_ID, "foo"); + options.addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, + UsernamePasswordToken.basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray()))); + request.setOptions(options); + Response response = getRestClient().performRequest(request); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + final AtomicReference lastClusterState = new AtomicReference<>(); + final boolean found = awaitSecurityAuditIndex(lastClusterState, QueryBuilders.matchQuery("opaque_id", "foo")); + + assertTrue("Did not find security audit index. Current cluster state:\n" + lastClusterState.get().toString(), found); + + SearchResponse searchResponse = client().prepareSearch(".security_audit_log*").setQuery( + QueryBuilders.matchQuery("opaque_id", "foo")).get(); + assertThat(searchResponse.getHits().getHits().length, greaterThan(0)); + + assertThat(searchResponse.getHits().getAt(0).getSourceAsMap().get("opaque_id"), is("foo")); + } + + private boolean awaitSecurityAuditIndex(AtomicReference lastClusterState, + QueryBuilder query) throws InterruptedException { final AtomicBoolean indexExists = new AtomicBoolean(false); - final boolean found = awaitBusy(() -> { + return awaitBusy(() -> { if (indexExists.get() == false) { ClusterState state = client().admin().cluster().prepareState().get().getState(); lastClusterState.set(state); @@ -138,28 +187,9 @@ public class IndexAuditIT extends ESIntegTestCase { logger.info("refreshing audit indices"); client().admin().indices().prepareRefresh(".security_audit_log*").get(); logger.info("refreshed audit indices"); - return client().prepareSearch(".security_audit_log*").setQuery(QueryBuilders.matchQuery("principal", USER)) - .get().getHits().getTotalHits() > 0; + return client().prepareSearch(".security_audit_log*").setQuery(query) + .get().getHits().getTotalHits() > 0; }, 60L, TimeUnit.SECONDS); - - assertTrue("Did not find security audit index. Current cluster state:\n" + lastClusterState.get().toString(), found); - - SearchResponse searchResponse = client().prepareSearch(".security_audit_log*").setQuery( - QueryBuilders.matchQuery("principal", USER)).get(); - assertThat(searchResponse.getHits().getHits().length, greaterThan(0)); - assertThat(searchResponse.getHits().getAt(0).getSourceAsMap().get("principal"), is(USER)); - } - - public void testAuditTrailTemplateIsRecreatedAfterDelete() throws Exception { - // this is already "tested" by the test framework since we wipe the templates before and after, - // but lets be explicit about the behavior - awaitIndexTemplateCreation(); - - // delete the template - DeleteIndexTemplateResponse deleteResponse = client().admin().indices() - .prepareDeleteTemplate(IndexAuditTrail.INDEX_TEMPLATE_NAME).execute().actionGet(); - assertThat(deleteResponse.isAcknowledged(), is(true)); - awaitIndexTemplateCreation(); } private void awaitIndexTemplateCreation() throws InterruptedException { From 9a7a649755c158977ec238920ed710b993ffceb4 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Wed, 25 Jul 2018 16:10:32 +0100 Subject: [PATCH 4/7] [ML][DOCS] Add documentation for detector rules and filters (#32013) --- x-pack/docs/build.gradle | 10 + x-pack/docs/en/ml/api-quickref.asciidoc | 9 + x-pack/docs/en/ml/configuring.asciidoc | 4 + .../docs/en/ml/detector-custom-rules.asciidoc | 230 ++++++++++++++++++ x-pack/docs/en/ml/functions/geo.asciidoc | 2 + x-pack/docs/en/ml/functions/metric.asciidoc | 4 +- x-pack/docs/en/ml/functions/rare.asciidoc | 2 + x-pack/docs/en/rest-api/defs.asciidoc | 3 + x-pack/docs/en/rest-api/ml-api.asciidoc | 12 + .../en/rest-api/ml/delete-filter.asciidoc | 53 ++++ .../en/rest-api/ml/filterresource.asciidoc | 16 ++ .../docs/en/rest-api/ml/get-filter.asciidoc | 84 +++++++ .../docs/en/rest-api/ml/jobresource.asciidoc | 73 +++++- .../docs/en/rest-api/ml/put-filter.asciidoc | 68 ++++++ .../en/rest-api/ml/update-filter.asciidoc | 67 +++++ .../docs/en/rest-api/ml/update-job.asciidoc | 23 +- 16 files changed, 648 insertions(+), 12 deletions(-) create mode 100644 x-pack/docs/en/ml/detector-custom-rules.asciidoc create mode 100644 x-pack/docs/en/rest-api/ml/delete-filter.asciidoc create mode 100644 x-pack/docs/en/rest-api/ml/filterresource.asciidoc create mode 100644 x-pack/docs/en/rest-api/ml/get-filter.asciidoc create mode 100644 x-pack/docs/en/rest-api/ml/put-filter.asciidoc create mode 100644 x-pack/docs/en/rest-api/ml/update-filter.asciidoc diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index 2ae26044be5..70f6061985a 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -309,6 +309,16 @@ setups['farequote_datafeed'] = setups['farequote_job'] + ''' "job_id":"farequote", "indexes":"farequote" } +''' +setups['ml_filter_safe_domains'] = ''' + - do: + xpack.ml.put_filter: + filter_id: "safe_domains" + body: > + { + "description": "A list of safe domains", + "items": ["*.google.com", "wikipedia.org"] + } ''' setups['server_metrics_index'] = ''' - do: diff --git a/x-pack/docs/en/ml/api-quickref.asciidoc b/x-pack/docs/en/ml/api-quickref.asciidoc index dc87a6ba209..be74167862e 100644 --- a/x-pack/docs/en/ml/api-quickref.asciidoc +++ b/x-pack/docs/en/ml/api-quickref.asciidoc @@ -47,6 +47,15 @@ The main {ml} resources can be accessed with a variety of endpoints: * {ref}/ml-delete-calendar-job.html[DELETE /calendars/+++/jobs/+++]: Disassociate a job from a calendar * {ref}/ml-delete-calendar.html[DELETE /calendars/+++]: Delete a calendar +[float] +[[ml-api-filters]] +=== /filters/ + +* {ref}/ml-put-filter.html[PUT /filters/+++]: Create a filter +* {ref}/ml-update-filter.html[POST /filters/+++/_update]: Update a filter +* {ref}/ml-get-filter.html[GET /filters/+++]: List filters +* {ref}/ml-delete-filter.html[DELETE /filter/+++]: Delete a filter + [float] [[ml-api-datafeeds]] === /datafeeds/ diff --git a/x-pack/docs/en/ml/configuring.asciidoc b/x-pack/docs/en/ml/configuring.asciidoc index c2c6e69a711..e35f046a82b 100644 --- a/x-pack/docs/en/ml/configuring.asciidoc +++ b/x-pack/docs/en/ml/configuring.asciidoc @@ -34,6 +34,7 @@ The scenarios in this section describe some best practices for generating useful * <> * <> * <> +* <> :edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/ml/customurl.asciidoc include::customurl.asciidoc[] @@ -49,3 +50,6 @@ include::populations.asciidoc[] :edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/ml/transforms.asciidoc include::transforms.asciidoc[] + +:edit_url: https://github.com/elastic/elasticsearch/edit/{branch}/x-pack/docs/en/ml/detector-custom-rules.asciidoc +include::detector-custom-rules.asciidoc[] \ No newline at end of file diff --git a/x-pack/docs/en/ml/detector-custom-rules.asciidoc b/x-pack/docs/en/ml/detector-custom-rules.asciidoc new file mode 100644 index 00000000000..18d516fae2d --- /dev/null +++ b/x-pack/docs/en/ml/detector-custom-rules.asciidoc @@ -0,0 +1,230 @@ +[role="xpack"] +[[ml-configuring-detector-custom-rules]] +=== Customizing detectors with rules and filters + +<> enable you to change the behavior of anomaly +detectors based on domain-specific knowledge. + +Rules describe _when_ a detector should take a certain _action_ instead +of following its default behavior. To specify the _when_ a rule uses +a `scope` and `conditions`. You can think of `scope` as the categorical +specification of a rule, while `conditions` are the numerical part. +A rule can have a scope, one or more conditions, or a combination of +scope and conditions. + +Let us see how those can be configured by examples. + +==== Specifying rule scope + +Let us assume we are configuring a job in order to DNS data exfiltration. +Our data contain fields "subdomain" and "highest_registered_domain". +We can use a detector that looks like `high_info_content(subdomain) over highest_registered_domain`. +If we run such a job it is possible that we discover a lot of anomalies on +frequently used domains that we have reasons to trust. As security analysts, we +are not interested in such anomalies. Ideally, we could instruct the detector to +skip results for domains that we consider safe. Using a rule with a scope allows +us to achieve this. + +First, we need to create a list with our safe domains. Those lists are called +`filters` in {ml}. Filters can be shared across jobs. + +We create our filter using the {ref}/ml-put-filter.html[put filter API]: + +[source,js] +---------------------------------- +PUT _xpack/ml/filters/safe_domains +{ + "description": "Our list of safe domains", + "items": ["safe.com", "trusted.com"] +} +---------------------------------- +// CONSOLE + +Now, we can create our job specifying a scope that uses the filter for the +`highest_registered_domain` field: + +[source,js] +---------------------------------- +PUT _xpack/ml/anomaly_detectors/dns_exfiltration_with_rule +{ + "analysis_config" : { + "bucket_span":"5m", + "detectors" :[{ + "function":"high_info_content", + "field_name": "subdomain", + "over_field_name": "highest_registered_domain", + "custom_rules": [{ + "actions": ["skip_result"], + "scope": { + "highest_registered_domain": { + "filter_id": "safe_domains", + "filter_type": "include" + } + } + }] + }] + }, + "data_description" : { + "time_field":"timestamp" + } +} +---------------------------------- +// CONSOLE + +As time advances and we see more data and more results, we might encounter new +domains that we want to add in the filter. We can do that by using the +{ref}/ml-update-filter.html[update filter API]: + +[source,js] +---------------------------------- +POST _xpack/ml/filters/safe_domains/_update +{ + "add_items": ["another-safe.com"] +} +---------------------------------- +// CONSOLE +// TEST[setup:ml_filter_safe_domains] + +Note that we can provide scope for any of the partition/over/by fields. +In the following example we scope multiple fields: + +[source,js] +---------------------------------- +PUT _xpack/ml/anomaly_detectors/scoping_multiple_fields +{ + "analysis_config" : { + "bucket_span":"5m", + "detectors" :[{ + "function":"count", + "partition_field_name": "my_partition", + "over_field_name": "my_over", + "by_field_name": "my_by", + "custom_rules": [{ + "actions": ["skip_result"], + "scope": { + "my_partition": { + "filter_id": "filter_1" + }, + "my_over": { + "filter_id": "filter_2" + }, + "my_by": { + "filter_id": "filter_3" + } + } + }] + }] + }, + "data_description" : { + "time_field":"timestamp" + } +} +---------------------------------- +// CONSOLE + +Such a detector will skip results when the values of all 3 scoped fields +are included in the referenced filters. + +==== Specifying rule conditions + +Imagine a detector that looks for anomalies in CPU utilization. +Given a machine that is idle for long enough, small movement in CPU could +result in anomalous results where the `actual` value is quite small, for +example, 0.02. Given our knowledge about how CPU utilization behaves we might +determine that anomalies with such small actual values are not interesting for +investigation. + +Let us now configure a job with a rule that will skip results where CPU +utilization is less than 0.20. + +[source,js] +---------------------------------- +PUT _xpack/ml/anomaly_detectors/cpu_with_rule +{ + "analysis_config" : { + "bucket_span":"5m", + "detectors" :[{ + "function":"high_mean", + "field_name": "cpu_utilization", + "custom_rules": [{ + "actions": ["skip_result"], + "conditions": [ + { + "applies_to": "actual", + "operator": "lt", + "value": 0.20 + } + ] + }] + }] + }, + "data_description" : { + "time_field":"timestamp" + } +} +---------------------------------- +// CONSOLE + +When there are multiple conditions they are combined with a logical `and`. +This is useful when we want the rule to apply to a range. We simply create +a rule with two conditions, one for each end of the desired range. + +Here is an example where a count detector will skip results when the count +is greater than 30 and less than 50: + +[source,js] +---------------------------------- +PUT _xpack/ml/anomaly_detectors/rule_with_range +{ + "analysis_config" : { + "bucket_span":"5m", + "detectors" :[{ + "function":"count", + "custom_rules": [{ + "actions": ["skip_result"], + "conditions": [ + { + "applies_to": "actual", + "operator": "gt", + "value": 30 + }, + { + "applies_to": "actual", + "operator": "lt", + "value": 50 + } + ] + }] + }] + }, + "data_description" : { + "time_field":"timestamp" + } +} +---------------------------------- +// CONSOLE + +==== Rules in the life-cycle of a job + +Rules only affect results created after the rules were applied. +Let us imagine that we have configured a job and it has been running +for some time. After observing its results we decide that we can employ +rules in order to get rid of some uninteresting results. We can use +the update-job API to do so. However, the rule we added will only be in effect +for any results created from the moment we added the rule onwards. Past results +will remain unaffected. + +==== Using rules VS filtering data + +It might appear like using rules is just another way of filtering the data +that feeds into a job. For example, a rule that skips results when the +partition field value is in a filter sounds equivalent to having a query +that filters out such documents. But it is not. There is a fundamental +difference. When the data is filtered before reaching a job it is as if they +never existed for the job. With rules, the data still reaches the job and +affects its behavior (depending on the rule actions). + +For example, a rule with the `skip_result` action means all data will still +be modeled. On the other hand, a rule with the `skip_model_update` action means +results will still be created even though the model will not be updated by +data matched by a rule. diff --git a/x-pack/docs/en/ml/functions/geo.asciidoc b/x-pack/docs/en/ml/functions/geo.asciidoc index e9685b46e16..5bcf6c33945 100644 --- a/x-pack/docs/en/ml/functions/geo.asciidoc +++ b/x-pack/docs/en/ml/functions/geo.asciidoc @@ -8,6 +8,8 @@ input data. The {xpackml} features include the following geographic function: `lat_long`. NOTE: You cannot create forecasts for jobs that contain geographic functions. +You also cannot add rules with conditions to detectors that use geographic +functions. [float] [[ml-lat-long]] diff --git a/x-pack/docs/en/ml/functions/metric.asciidoc b/x-pack/docs/en/ml/functions/metric.asciidoc index 3ee51797027..9d6f3515a02 100644 --- a/x-pack/docs/en/ml/functions/metric.asciidoc +++ b/x-pack/docs/en/ml/functions/metric.asciidoc @@ -15,6 +15,9 @@ The {xpackml} features include the following metric functions: * <> * xref:ml-metric-varp[`varp`, `high_varp`, `low_varp`] +NOTE: You cannot add rules with conditions to detectors that use the `metric` +function. + [float] [[ml-metric-min]] ==== Min @@ -221,7 +224,6 @@ mean `responsetime` for each application over time. It detects when the mean The `metric` function combines `min`, `max`, and `mean` functions. You can use it as a shorthand for a combined analysis. If you do not specify a function in a detector, this is the default function. -//TBD: Is that default behavior still true? High- and low-sided functions are not applicable. You cannot use this function when a `summary_count_field_name` is specified. diff --git a/x-pack/docs/en/ml/functions/rare.asciidoc b/x-pack/docs/en/ml/functions/rare.asciidoc index fc30918b508..1531285a7ad 100644 --- a/x-pack/docs/en/ml/functions/rare.asciidoc +++ b/x-pack/docs/en/ml/functions/rare.asciidoc @@ -15,6 +15,8 @@ number of times (frequency) rare values occur. `exclude_frequent`. * You cannot create forecasts for jobs that contain `rare` or `freq_rare` functions. +* You cannot add rules with conditions to detectors that use `rare` or +`freq_rare` functions. * Shorter bucket spans (less than 1 hour, for example) are recommended when looking for rare events. The functions model whether something happens in a bucket at least once. With longer bucket spans, it is more likely that diff --git a/x-pack/docs/en/rest-api/defs.asciidoc b/x-pack/docs/en/rest-api/defs.asciidoc index 99600472a09..349ce343c7a 100644 --- a/x-pack/docs/en/rest-api/defs.asciidoc +++ b/x-pack/docs/en/rest-api/defs.asciidoc @@ -8,6 +8,7 @@ job configuration options. * <> * <> * <> +* <> * <> * <> * <> @@ -19,6 +20,8 @@ include::ml/calendarresource.asciidoc[] [role="xpack"] include::ml/datafeedresource.asciidoc[] [role="xpack"] +include::ml/filterresource.asciidoc[] +[role="xpack"] include::ml/jobresource.asciidoc[] [role="xpack"] include::ml/jobcounts.asciidoc[] diff --git a/x-pack/docs/en/rest-api/ml-api.asciidoc b/x-pack/docs/en/rest-api/ml-api.asciidoc index e9a987cc4a7..b48e9f93404 100644 --- a/x-pack/docs/en/rest-api/ml-api.asciidoc +++ b/x-pack/docs/en/rest-api/ml-api.asciidoc @@ -15,6 +15,14 @@ machine learning APIs and in advanced job configuration options in Kibana. * <>, <> * <>, <> +[float] +[[ml-api-filter-endpoint]] +=== Filters + +* <>, <> +* <> +* <> + [float] [[ml-api-datafeed-endpoint]] === {dfeeds-cap} @@ -69,11 +77,13 @@ include::ml/close-job.asciidoc[] //CREATE include::ml/put-calendar.asciidoc[] include::ml/put-datafeed.asciidoc[] +include::ml/put-filter.asciidoc[] include::ml/put-job.asciidoc[] //DELETE include::ml/delete-calendar.asciidoc[] include::ml/delete-datafeed.asciidoc[] include::ml/delete-calendar-event.asciidoc[] +include::ml/delete-filter.asciidoc[] include::ml/delete-job.asciidoc[] include::ml/delete-calendar-job.asciidoc[] include::ml/delete-snapshot.asciidoc[] @@ -93,6 +103,7 @@ include::ml/get-job.asciidoc[] include::ml/get-job-stats.asciidoc[] include::ml/get-snapshot.asciidoc[] include::ml/get-calendar-event.asciidoc[] +include::ml/get-filter.asciidoc[] include::ml/get-record.asciidoc[] //OPEN include::ml/open-job.asciidoc[] @@ -107,6 +118,7 @@ include::ml/start-datafeed.asciidoc[] include::ml/stop-datafeed.asciidoc[] //UPDATE include::ml/update-datafeed.asciidoc[] +include::ml/update-filter.asciidoc[] include::ml/update-job.asciidoc[] include::ml/update-snapshot.asciidoc[] //VALIDATE diff --git a/x-pack/docs/en/rest-api/ml/delete-filter.asciidoc b/x-pack/docs/en/rest-api/ml/delete-filter.asciidoc new file mode 100644 index 00000000000..b58d2980b88 --- /dev/null +++ b/x-pack/docs/en/rest-api/ml/delete-filter.asciidoc @@ -0,0 +1,53 @@ +[role="xpack"] +[[ml-delete-filter]] +=== Delete Filter API +++++ +Delete Filter +++++ + +Deletes a filter. + + +==== Request + +`DELETE _xpack/ml/filters/` + + +==== Description + +This API deletes a {stack-ov}/ml-rules.html[filter]. +If a {ml} job references the filter, you cannot delete the filter. You must +update or delete the job before you can delete the filter. + + +==== Path Parameters + +`filter_id` (required):: + (string) Identifier for the filter. + + +==== Authorization + +You must have `manage_ml`, or `manage` cluster privileges to use this API. +For more information, see {xpack-ref}/security-privileges.html[Security Privileges]. + + +==== Examples + +The following example deletes the `safe_domains` filter: + +[source,js] +-------------------------------------------------- +DELETE _xpack/ml/filters/safe_domains +-------------------------------------------------- +// CONSOLE +// TEST[setup:ml_filter_safe_domains] + +When the filter is deleted, you receive the following results: +[source,js] +---- +{ + "acknowledged": true +} +---- +//TESTRESPONSE diff --git a/x-pack/docs/en/rest-api/ml/filterresource.asciidoc b/x-pack/docs/en/rest-api/ml/filterresource.asciidoc new file mode 100644 index 00000000000..64768da4911 --- /dev/null +++ b/x-pack/docs/en/rest-api/ml/filterresource.asciidoc @@ -0,0 +1,16 @@ +[role="xpack"] +[[ml-filter-resource]] +=== Filter Resources + +A filter resource has the following properties: + +`filter_id`:: + (string) A string that uniquely identifies the filter. + +`description`:: + (array) A description of the filter. + +`items`:: + (array of strings) An array of strings which is the filter item list. + +For more information, see {stack-ov}/ml-rules.html[Machine learning rules and filters]. diff --git a/x-pack/docs/en/rest-api/ml/get-filter.asciidoc b/x-pack/docs/en/rest-api/ml/get-filter.asciidoc new file mode 100644 index 00000000000..89f40cf3312 --- /dev/null +++ b/x-pack/docs/en/rest-api/ml/get-filter.asciidoc @@ -0,0 +1,84 @@ +[role="xpack"] +[[ml-get-filter]] +=== Get Filters API +++++ +Get Filters +++++ + +Retrieves filters. + + +==== Request + +`GET _xpack/ml/filters/` + + +`GET _xpack/ml/filters/` + + +===== Description + +You can get a single filter or all filters. For more information, see +{stack-ov}/ml-rules.html[Machine learning rules and filters]. + + +==== Path Parameters + +`filter_id`:: + (string) Identifier for the filter. + + +==== Request Body + +`from`::: + (integer) Skips the specified number of filters. + +`size`::: + (integer) Specifies the maximum number of filters to obtain. + + +==== Results + +The API returns the following information: + +`filters`:: + (array) An array of filter resources. + For more information, see <>. + + +==== Authorization + +You must have `monitor_ml`, `monitor`, `manage_ml`, or `manage` cluster +privileges to use this API. For more information, see +{xpack-ref}/security-privileges.html[Security Privileges]. + + +==== Examples + +The following example gets configuration information for the `safe_domains` +filter: + +[source,js] +-------------------------------------------------- +GET _xpack/ml/filters/safe_domains +-------------------------------------------------- +// CONSOLE +// TEST[setup:ml_filter_safe_domains] + +The API returns the following results: +[source,js] +---- +{ + "count": 1, + "filters": [ + { + "filter_id": "safe_domains", + "description": "A list of safe domains", + "items": [ + "*.google.com", + "wikipedia.org" + ] + } + ] +} +---- +//TESTRESPONSE diff --git a/x-pack/docs/en/rest-api/ml/jobresource.asciidoc b/x-pack/docs/en/rest-api/ml/jobresource.asciidoc index bb959fd728c..e4b6da11dc5 100644 --- a/x-pack/docs/en/rest-api/ml/jobresource.asciidoc +++ b/x-pack/docs/en/rest-api/ml/jobresource.asciidoc @@ -245,7 +245,7 @@ NOTE: The `field_name` cannot contain double quotes or backslashes. -- `function`:: - (string) The analysis function that is used. + (string) The analysis function that is used. For example, `count`, `rare`, `mean`, `min`, `max`, and `sum`. For more information, see {xpack-ref}/ml-functions.html[Function Reference]. @@ -262,7 +262,12 @@ NOTE: The `field_name` cannot contain double quotes or backslashes. `use_null`:: (boolean) Defines whether a new series is used as the null series - when there is no value for the by or partition fields. The default value is `false`. + + when there is no value for the by or partition fields. The default value is `false`. + +`custom_rules`:: + (array) An array of rule objects, which enable customizing how the detector works. + For example, a rule may dictate to the detector conditions under which results should be skipped. + For more information see <>. + + -- IMPORTANT: Field names are case sensitive, for example a field named 'Bytes' @@ -270,9 +275,9 @@ is different from one named 'bytes'. -- -After you create a job, the only property you can change in the detector -configuration object is the `detector_description`; all other properties are -informational. +After you create a job, the only properties you can change in the detector +configuration object are the `detector_description` and the `custom_rules`; +all other properties are informational. [float] [[ml-datadescription]] @@ -408,6 +413,64 @@ the categorization field value came from. For more information, see {xpack-ref}/ml-configuring-categories.html[Categorizing Log Messages]. +[float] +[[ml-detector-custom-rule]] +==== Detector Custom Rule + +{stack-ov}/ml-rules.html[Custom rules] enable you to customize the way detectors +operate. + +A rule has the following properties: + +`actions`:: + (array) The set of actions to be triggered when the rule applies. + If more than one action is specified the effects of all actions are combined. + The available actions include: + + `skip_result`::: The result will not be created. This is the default value. + Unless you also specify `skip_model_update`, the model will be updated as + usual with the corresponding series value. + `skip_model_update`::: The value for that series will not be used to update + the model. Unless you also specify `skip_result`, the results will be created + as usual. This action is suitable when certain values are expected to be + consistently anomalous and they affect the model in a way that negatively + impacts the rest of the results. +`scope`:: + (object) An optional scope of series where the rule applies. By default the scope + includes all series. Scoping is allowed for any of the partition/by/over fields. + To add a scope for a field add the field name as a key in the scope object and + set its value to an object with properties: + `filter_id`:: + (string) The id of the <> to be used. + `filter_type`:: + (string) Either `include` (the rule applies for values in the filter) + or `exclude` (the rule applies for values not in the filter). Defaults + to `include`. + +`conditions`:: + (array) An optional array of numeric conditions when the rule applies. + Multiple conditions are combined together with a logical `AND`. ++ +-- +NOTE: If your detector uses `lat_long`, `metric`, `rare`, or `freq_rare` +functions, you cannot specify `conditions` for your rule. + + +A condition has the following properties: + +`applies_to`::: + (string) Specifies the result property to which the condition applies. + The available options are `actual`, `typical`, `diff_from_typical`, `time`. +`operator`::: + (string) Specifies the condition operator. The available options are + `gt` (greater than), `gte` (greater than or equals), `lt` (less than) and `lte` (less than or equals). +`value`::: + (double) The value that is compared against the `applied_to` field using the `operator`. +-- + +A rule is required to either have a non-empty scope or at least one condition. +For more examples see +{stack-ov}/ml-configuring-detector-custom-rules.html[Configuring Detector Custom Rules]. + [float] [[ml-apilimits]] ==== Analysis Limits diff --git a/x-pack/docs/en/rest-api/ml/put-filter.asciidoc b/x-pack/docs/en/rest-api/ml/put-filter.asciidoc new file mode 100644 index 00000000000..d2982a56f61 --- /dev/null +++ b/x-pack/docs/en/rest-api/ml/put-filter.asciidoc @@ -0,0 +1,68 @@ +[role="xpack"] +[[ml-put-filter]] +=== Create Filter API +++++ +Create Filter +++++ + +Instantiates a filter. + +==== Request + +`PUT _xpack/ml/filters/` + +===== Description + +A {stack-ov}/ml-rules.html[filter] contains a list of strings. +It can be used by one or more jobs. Specifically, filters are referenced in +the `custom_rules` property of <>. + +==== Path Parameters + +`filter_id` (required):: + (string) Identifier for the filter. + + +==== Request Body + +`description`:: + (string) A description of the filter. + +`items`:: + (array of strings) The items of the filter. + A wildcard `*` can be used at the beginning + or the end of an item. Up to 10000 items + are allowed in each filter. + + +==== Authorization + +You must have `manage_ml`, or `manage` cluster privileges to use this API. +For more information, see +{xpack-ref}/security-privileges.html[Security Privileges]. + + +==== Examples + +The following example creates the `safe_domains` filter: + +[source,js] +-------------------------------------------------- +PUT _xpack/ml/filters/safe_domains +{ + "description": "A list of safe domains", + "items": ["*.google.com", "wikipedia.org"] +} +-------------------------------------------------- +// CONSOLE + +When the filter is created, you receive the following response: +[source,js] +---- +{ + "filter_id": "safe_domains", + "description": "A list of safe domains", + "items": ["*.google.com", "wikipedia.org"] +} +---- +//TESTRESPONSE diff --git a/x-pack/docs/en/rest-api/ml/update-filter.asciidoc b/x-pack/docs/en/rest-api/ml/update-filter.asciidoc new file mode 100644 index 00000000000..1b6760dfed6 --- /dev/null +++ b/x-pack/docs/en/rest-api/ml/update-filter.asciidoc @@ -0,0 +1,67 @@ +[role="xpack"] +[[ml-update-filter]] +=== Update Filter API +++++ +Update Filter +++++ + +Updates the description of a filter, adds items, or removes items. + +==== Request + +`POST _xpack/ml/filters//_update` + +//==== Description + +==== Path Parameters + +`filter_id` (required):: + (string) Identifier for the filter. + + +==== Request Body + +`description`:: + (string) A description for the filter. See <>. + +`add_items`:: + (array of strings) The items to add to the filter. + +`remove_items`:: + (array of strings) The items to remove from the filter. + + +==== Authorization + +You must have `manage_ml`, or `manage` cluster privileges to use this API. +For more information, see +{xpack-ref}/security-privileges.html[Security Privileges]. + + +==== Examples + +You can change the description, add and remove items to the `safe_domains` filter as follows: + +[source,js] +-------------------------------------------------- +POST _xpack/ml/filters/safe_domains/_update +{ + "description": "Updated list of domains", + "add_items": ["*.myorg.com"], + "remove_items": ["wikipedia.org"] +} +-------------------------------------------------- +// CONSOLE +// TEST[setup:ml_filter_safe_domains] + +The API returns the following results: + +[source,js] +---- +{ + "filter_id": "safe_domains", + "description": "Updated list of domains", + "items": ["*.google.com", "*.myorg.com"] +} +---- +//TESTRESPONSE diff --git a/x-pack/docs/en/rest-api/ml/update-job.asciidoc b/x-pack/docs/en/rest-api/ml/update-job.asciidoc index 23046febc2e..852745e9dd9 100644 --- a/x-pack/docs/en/rest-api/ml/update-job.asciidoc +++ b/x-pack/docs/en/rest-api/ml/update-job.asciidoc @@ -35,6 +35,8 @@ each periodic persistence of the model. See <>. | Yes |`description` |A description of the job. See <>. | No +|`detectors` |An array of <>. | No + |`groups` |A list of job groups. See <>. | No |`model_plot_config`: `enabled` |If true, enables calculation and storage of the @@ -58,12 +60,6 @@ if the job is open when you make the update, you must stop the data feed, close the job, then restart the data feed and open the job for the changes to take effect. -//|`analysis_config`: `detectors`: `detector_index` | A unique identifier of the -//detector. Matches the order of detectors returned by -//<>, starting from 0. | No -//|`analysis_config`: `detectors`: `detector_description` |A description of the -//detector. See <>. | No - [NOTE] -- * You can update the `analysis_limits` only while the job is closed. @@ -73,6 +69,21 @@ of `hard_limit`, this means that it was unable to process some data. You might want to re-run this job with an increased `model_memory_limit`. -- +[[ml-detector-update]] +==== Detector Update Objects + +A detector update object has the following properties: + +`detector_index`:: + (integer) The identifier of the detector to update. + +`description`:: + (string) The new description for the detector. + +`custom_rules`:: + (array) The new list of <> for the detector. + +No other detector property can be updated. ==== Authorization From 6832aa6797c3105747554d9e21213f7d2a15e344 Mon Sep 17 00:00:00 2001 From: lcawl Date: Tue, 24 Jul 2018 17:06:46 -0700 Subject: [PATCH 5/7] [DOCS] Adds link from bucket_span property to common time units --- x-pack/docs/en/rest-api/ml/jobresource.asciidoc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/x-pack/docs/en/rest-api/ml/jobresource.asciidoc b/x-pack/docs/en/rest-api/ml/jobresource.asciidoc index e4b6da11dc5..3c4e330722f 100644 --- a/x-pack/docs/en/rest-api/ml/jobresource.asciidoc +++ b/x-pack/docs/en/rest-api/ml/jobresource.asciidoc @@ -106,7 +106,8 @@ An analysis configuration object has the following properties: `bucket_span`:: (time units) The size of the interval that the analysis is aggregated into, - typically between `5m` and `1h`. The default value is `5m`. + typically between `5m` and `1h`. The default value is `5m`. For more + information about time units, see <>. `categorization_field_name`:: (string) If this property is specified, the values of the specified field will @@ -160,8 +161,7 @@ no analysis can occur and an error is returned. (time units) The size of the window in which to expect data that is out of time order. The default value is 0 (no latency). If you specify a non-zero value, it must be greater than or equal to one second. For more information - about time units, see - {ref}/common-options.html#time-units[Time Units]. + about time units, see <>. + -- NOTE: Latency is only applicable when you send data by using @@ -511,8 +511,7 @@ Specifying a string is recommended for clarity. If you specify a byte size unit of `b` or `kb` and the number does not equate to a discrete number of megabytes, it is rounded down to the closest MiB. The minimum valid value is 1 MiB. If you specify a value less than 1 MiB, an error occurs. For more information about -supported byte size units, see -{ref}/common-options.html#byte-units[Byte size units]. +supported byte size units, see <>. If your `elasticsearch.yml` file contains an `xpack.ml.max_model_memory_limit` setting, an error occurs when you try to create jobs that have From 5a12c63b4e98c9ef95baa6cf2e932837e7031420 Mon Sep 17 00:00:00 2001 From: lcawl Date: Tue, 24 Jul 2018 16:11:56 -0700 Subject: [PATCH 6/7] [DOCS] Fixes typo in ML aggregations page --- x-pack/docs/en/ml/aggregations.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/docs/en/ml/aggregations.asciidoc b/x-pack/docs/en/ml/aggregations.asciidoc index 5ff54b76f01..07f46501569 100644 --- a/x-pack/docs/en/ml/aggregations.asciidoc +++ b/x-pack/docs/en/ml/aggregations.asciidoc @@ -105,8 +105,8 @@ For all other aggregations, if the aggregation name doesn't match the field name there are limitations in the drill-down functionality within the {ml} page in {kib}. -{dfeeds} support complex nested aggregations, this example uses the `derivative` -pipeline aggregation to find the 1st order derivative of the counter +{dfeeds-cap} support complex nested aggregations, this example uses the `derivative` +pipeline aggregation to find the first order derivative of the counter `system.network.out.bytes` for each value of the field `beat.name`. [source,js] From 6cf7588c3d8e7d79c77c9fd3fd091bc7c11f512d Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Wed, 25 Jul 2018 11:34:26 -0400 Subject: [PATCH 7/7] [TEST] Fix failure due to exception message in java11 (#32321) Java 11 uses more verbose exceptions messages, causing this assertion to fail. Changed the test to be less restrictive and only look for the classes we care about. --- .../xpack/rollup/RollupResponseTranslationTests.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java index 98e3ad8197a..35d9f0d133a 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupResponseTranslationTests.java @@ -508,10 +508,12 @@ public class RollupResponseTranslationTests extends AggregatorTestCase { BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()); ScriptService scriptService = mock(ScriptService.class); InternalAggregation.ReduceContext reduceContext = new InternalAggregation.ReduceContext(bigArrays, scriptService, true); - Exception e = expectThrows(RuntimeException.class, + ClassCastException e = expectThrows(ClassCastException.class, () -> RollupResponseTranslator.combineResponses(msearch, reduceContext)); - assertThat(e.getMessage(), equalTo("org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds " + - "cannot be cast to org.elasticsearch.search.aggregations.InternalMultiBucketAggregation")); + assertThat(e.getMessage(), + containsString("org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds")); + assertThat(e.getMessage(), + containsString("org.elasticsearch.search.aggregations.InternalMultiBucketAggregation")); } public void testDateHisto() throws IOException {