From c9aa4e3d59bf7aca589e0ed4488ae97f6f1a56b9 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Fri, 12 Jan 2018 14:26:04 -0700 Subject: [PATCH 01/15] [Monitoring] Add additional metric fields to Beats template (elastic/x-pack-elasticsearch#3337) * [Monitoring] Add uptime.ms field to Beats template * Add cpu mapping [WIP] * sort fields * hierarchical - no dots in fields * add rss, more event metrics, system cpu and os load * fix some mapping errors Original commit: elastic/x-pack-elasticsearch@59cbcb4ee5d01aa7bcc8ede668ce47bafeb7ceb0 --- .../src/main/resources/monitoring-beats.json | 199 +++++++++++++----- 1 file changed, 148 insertions(+), 51 deletions(-) diff --git a/plugin/src/main/resources/monitoring-beats.json b/plugin/src/main/resources/monitoring-beats.json index 3541ffd2bcd..d2cc514d5a8 100644 --- a/plugin/src/main/resources/monitoring-beats.json +++ b/plugin/src/main/resources/monitoring-beats.json @@ -1,74 +1,37 @@ { - "index_patterns": [ ".monitoring-beats-${monitoring.template.version}-*" ], - "version": 7000001, + "index_patterns": [ + ".monitoring-beats-${monitoring.template.version}-*" + ], "settings": { - "index.number_of_shards": 1, - "index.number_of_replicas": 0, "index.auto_expand_replicas": "0-1", + "index.codec": "best_compression", "index.format": 6, - "index.codec": "best_compression" + "index.number_of_replicas": 0, + "index.number_of_shards": 1 }, + "version": 7000001, "mappings": { "doc": { "dynamic": false, "properties": { - "cluster_uuid": { - "type": "keyword" - }, - "timestamp": { - "type": "date", - "format": "date_time" - }, - "interval_ms": { - "type": "long" - }, - "type": { - "type": "keyword" - }, - "source_node": { - "properties": { - "uuid": { - "type": "keyword" - }, - "host": { - "type": "keyword" - }, - "transport_address": { - "type": "keyword" - }, - "ip": { - "type": "keyword" - }, - "name": { - "type": "keyword" - } - } - }, "beats_stats": { "properties": { - "timestamp": { - "type": "date", - "format": "date_time" - }, - "tags": { - "type": "keyword" - }, "beat": { "properties": { - "uuid": { + "host": { + "type": "keyword" + }, + "name": { "type": "keyword" }, "type": { "type": "keyword" }, + "uuid": { + "type": "keyword" + }, "version": { "type": "keyword" - }, - "host": { - "type": "keyword" - }, - "name": { - "type": "keyword" } } }, @@ -76,6 +39,41 @@ "properties": { "beat": { "properties": { + "cpu": { + "properties": { + "total": { + "properties": { + "norm": { + "properties": { + "pct": { + "type": "double" + } + } + }, + "pct": { + "type": "double" + }, + "value": { + "type": "double" + } + } + } + } + }, + "info": { + "properties": { + "ephemeral_id": { + "type": "keyword" + }, + "uptime": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, "memstats": { "properties": { "gc_next": { @@ -86,6 +84,9 @@ }, "memory_total": { "type": "long" + }, + "rss": { + "type": "long" } } } @@ -126,6 +127,12 @@ "batches": { "type": "long" }, + "dropped": { + "type": "long" + }, + "duplicates": { + "type": "long" + }, "failed": { "type": "long" }, @@ -199,10 +206,100 @@ } } } + }, + "system": { + "properties": { + "cpu": { + "properties": { + "cores": { + "type": "long" + }, + "total": { + "properties": { + "norm": { + "properties": { + "pct": { + "type": "double" + } + } + }, + "pct": { + "type": "double" + } + } + } + } + }, + "load": { + "properties": { + "1": { + "type": "double" + }, + "15": { + "type": "double" + }, + "5": { + "type": "double" + }, + "norm": { + "properties": { + "1": { + "type": "double" + }, + "15": { + "type": "double" + }, + "5": { + "type": "double" + } + } + } + } + } + } } } + }, + "tags": { + "type": "keyword" + }, + "timestamp": { + "format": "date_time", + "type": "date" } } + }, + "cluster_uuid": { + "type": "keyword" + }, + "interval_ms": { + "type": "long" + }, + "source_node": { + "properties": { + "host": { + "type": "keyword" + }, + "ip": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "transport_address": { + "type": "keyword" + }, + "uuid": { + "type": "keyword" + } + } + }, + "timestamp": { + "format": "date_time", + "type": "date" + }, + "type": { + "type": "keyword" } } } From 56b0f28aa3b2132753527f3125537aba11ed5749 Mon Sep 17 00:00:00 2001 From: lcawley Date: Fri, 12 Jan 2018 13:52:19 -0800 Subject: [PATCH 02/15] [DOCS] More broken link fixes Original commit: elastic/x-pack-elasticsearch@dd52976660854014f6e2a46471466d4859d494cd --- .../security/securing-communications/setting-up-ssl.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/security/securing-communications/setting-up-ssl.asciidoc b/docs/en/security/securing-communications/setting-up-ssl.asciidoc index a38ccf0e38d..ff8da9a29ff 100644 --- a/docs/en/security/securing-communications/setting-up-ssl.asciidoc +++ b/docs/en/security/securing-communications/setting-up-ssl.asciidoc @@ -42,9 +42,9 @@ See <>. [[installing-node-certificates]] ==== Node Certificates -See {ref}/node-certificates.html[Generating Node Certificates]. +See {ref}/configuring-tls.html#node-certificates[Generating Node Certificates]. [[enable-ssl]] ==== Enabling TLS in the Node Configuration -See {ref}/enable-ssl.html[Enabling TLS on {es} Nodes]. +See {ref}/configuring-tls.html#enable-ssl[Enabling TLS on {es} Nodes]. From d4cddc12d0f5607d04220d45c19dd8862af7e9c4 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Mon, 15 Jan 2018 10:28:03 +0100 Subject: [PATCH 03/15] Tests: Add @AwaitsFix annotation to flaky tests These contribute to unstable builds, and should be reenabled on a case by case base and worked on. Original commit: elastic/x-pack-elasticsearch@a15338415ac1bd06fdebad1bf93c5e2fec45de5c --- .../xpack/watcher/test/integration/BootStrapTests.java | 2 ++ .../elasticsearch/smoketest/MonitoringWithWatcherRestIT.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java index 3c98176ff73..17132b05392 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/BootStrapTests.java @@ -148,6 +148,7 @@ public class BootStrapTests extends AbstractWatcherIntegrationTestCase { assertThat(response.getWatchesCount(), equalTo(1L)); } + @AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/1915") public void testLoadExistingWatchesUponStartup() throws Exception { ensureWatcherStarted(); stopWatcher(); @@ -226,6 +227,7 @@ public class BootStrapTests extends AbstractWatcherIntegrationTestCase { assertSingleExecutionAndCompleteWatchHistory(numWatches, numRecords); } + @AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/3437") public void testTriggeredWatchLoading() throws Exception { createIndex("output"); client().prepareIndex("my-index", "foo", "bar") diff --git a/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java b/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java index 266a09ab30c..6f06e83875b 100644 --- a/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java +++ b/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.smoketest; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.client.Response; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.junit.annotations.TestLogging; @@ -28,6 +29,7 @@ import static org.elasticsearch.xpack.watcher.trigger.schedule.IntervalSchedule. import static org.hamcrest.Matchers.is; @TestLogging("org.elasticsearch.client:TRACE,tracer:TRACE") +@AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/2920") public class MonitoringWithWatcherRestIT extends ESRestTestCase { @After From e9dafbd78dfc39bf5a7ad2912eb4f2414b8e32ba Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 15 Jan 2018 15:47:19 +0000 Subject: [PATCH 04/15] [DOCS] Add documentation for ML categorization_analyzer (elastic/x-pack-elasticsearch#3554) This is the documentation for the changes made in elastic/x-pack-elasticsearch#3372. Relates elastic/machine-learning-cpp#491 Original commit: elastic/x-pack-elasticsearch@7d67e9d89446bedab878dc8306f746d2c1b9929a --- docs/en/ml/categories.asciidoc | 123 +++++++++++++++++++++++ docs/en/ml/limitations.asciidoc | 10 +- docs/en/rest-api/ml/jobresource.asciidoc | 108 ++++++++++++++++++++ 3 files changed, 238 insertions(+), 3 deletions(-) diff --git a/docs/en/ml/categories.asciidoc b/docs/en/ml/categories.asciidoc index d2c1ac2503f..8a7114e418e 100644 --- a/docs/en/ml/categories.asciidoc +++ b/docs/en/ml/categories.asciidoc @@ -79,6 +79,129 @@ image::images/ml-category-advanced.jpg["Advanced job configuration options relat NOTE: To add the `categorization_examples_limit` property, you must use the **Edit JSON** tab and copy the `analysis_limits` object from the API example. +It is possible to customize the way the categorization field values are interpreted +to an even greater extent: + +[source,js] +---------------------------------- +PUT _xpack/ml/anomaly_detectors/it_ops_new_logs2 +{ + "description" : "IT Ops Application Logs", + "analysis_config" : { + "categorization_field_name": "message", + "bucket_span":"30m", + "detectors" :[{ + "function":"count", + "by_field_name": "mlcategory", + "detector_description": "Unusual message counts" + }], + "categorization_analyzer":{ + "char_filter": [ + { "type": "pattern_replace", "pattern": "\\[statement:.*\\]" } <1> + ], + "tokenizer": "ml_classic", <2> + "filter": [ + { "type" : "stop", "stopwords": [ + "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday", + "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun", + "January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December", + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", + "GMT", "UTC" + ] } <3> + ] + } + }, + "analysis_limits":{ + "categorization_examples_limit": 5 + }, + "data_description" : { + "time_field":"time", + "time_format": "epoch_ms" + } +} +---------------------------------- +//CONSOLE +<1> The +{ref}/analysis-pattern-replace-charfilter.html[`pattern_replace` character filter] +here achieves exactly the same as the `categorization_filters` in the first +example. +<2> The `ml_classic` tokenizer works like the non-customizable tokenization +that was used for categorization in older versions of machine learning. Use +it if you want the same categorization behavior as older versions. +<3> English day/month words are filtered by default from log messages +before categorization. If your logs are in a different language and contain +dates then you may get better results by filtering day/month words in your +language. + +The optional `categorization_analyzer` property allows even greater customization +of how categorization interprets the categorization field value. It can refer to +a built-in Elasticsearch analyzer, or a combination of zero or more character +filters, a tokenizer, and zero or more token filters. + +The `ml_classic` tokenizer and the day/month stopword filter are more-or-less +equivalent to the following analyzer defined using only built-in Elasticsearch +{ref}/analysis-tokenizers.html[tokenizers] and +{ref}/analysis-tokenfilters.html[token filters]: + +[source,js] +---------------------------------- +PUT _xpack/ml/anomaly_detectors/it_ops_new_logs3 +{ + "description" : "IT Ops Application Logs", + "analysis_config" : { + "categorization_field_name": "message", + "bucket_span":"30m", + "detectors" :[{ + "function":"count", + "by_field_name": "mlcategory", + "detector_description": "Unusual message counts" + }], + "categorization_analyzer":{ + "tokenizer": { + "type" : "simple_pattern_split", + "pattern" : "[^-0-9A-Za-z_.]+" <1> + }, + "filter": [ + { "type" : "pattern_replace", "pattern": "^[0-9].*" }, <2> + { "type" : "pattern_replace", "pattern": "^[-0-9A-Fa-f.]+$" }, <3> + { "type" : "pattern_replace", "pattern": "^[^0-9A-Za-z]+" }, <4> + { "type" : "pattern_replace", "pattern": "[^0-9A-Za-z]+$" }, <5> + { "type" : "stop", "stopwords": [ + "", + "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday", + "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun", + "January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December", + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", + "GMT", "UTC" + ] } + ] + } + }, + "analysis_limits":{ + "categorization_examples_limit": 5 + }, + "data_description" : { + "time_field":"time", + "time_format": "epoch_ms" + } +} +---------------------------------- +//CONSOLE +<1> Tokens basically consist of hyphens, digits, letters, underscores and dots. +<2> By default categorization ignores tokens that begin with a digit. +<3> By default categorization also ignores tokens that are hexadecimal numbers. +<4> Underscores, hypens and dots are removed from the beginning of tokens. +<5> Also at the end of tokens. + +The key difference between the default `categorization_analyzer` and this example +analyzer is that using the `ml_classic` tokenizer is several times faster. (The +difference in behavior is that this custom analyzer does not include accented +letters in tokens whereas the `ml_classic` tokenizer will, although that could be +fixed by using more complex regular expressions.) + +NOTE: To add the `categorization_analyzer` property, you must use the **Edit JSON** +tab and copy the `categorization_analyzer` object from one of the API examples above. + After you open the job and start the {dfeed} or supply data to the job, you can view the results in {kib}. For example: diff --git a/docs/en/ml/limitations.asciidoc b/docs/en/ml/limitations.asciidoc index 1d8eda58bdf..242d4be4431 100644 --- a/docs/en/ml/limitations.asciidoc +++ b/docs/en/ml/limitations.asciidoc @@ -5,11 +5,15 @@ The following limitations and known problems apply to the {version} release of {xpack}: [float] -=== Categorization uses English tokenization rules and dictionary words +=== Categorization uses English dictionary words //See x-pack-elasticsearch/#3021 Categorization identifies static parts of unstructured logs and groups similar -messages together. This is currently supported only for English language log -messages. +messages together. The default categorization tokenizer assumes English language +log messages. For other languages you must define a different +categorization_analyzer for your job. Additionally, a dictionary used to influence +the categorization process contains only English words. This means categorization +may work better in English than in other languages. The ability to customize the +dictionary will be added in a future release. [float] === Pop-ups must be enabled in browsers diff --git a/docs/en/rest-api/ml/jobresource.asciidoc b/docs/en/rest-api/ml/jobresource.asciidoc index 7f20830ea23..2f3026fff08 100644 --- a/docs/en/rest-api/ml/jobresource.asciidoc +++ b/docs/en/rest-api/ml/jobresource.asciidoc @@ -110,6 +110,18 @@ An analysis configuration object has the following properties: consideration for defining categories. For example, you can exclude SQL statements that appear in your log files. For more information, see {xpack-ref}/ml-configuring-categories.html[Categorizing Log Messages]. + This property cannot be used at the same time as `categorization_analyzer`. + If you only want to define simple regular expression filters to be applied + prior to tokenization then it is easiest to specify them using this property. + If you also want to customize the tokenizer or post-tokenization filtering + then these filters must be included in the `categorization_analyzer` as + `pattern_replace` `char_filter`s. The effect is exactly the same. +//<>. + +`categorization_analyzer`:: + (object or string) If `categorization_field_name` is specified, + you can also define the analyzer that will be used to interpret the field + to be categorized. See <>. //<>. `detectors`:: @@ -293,6 +305,102 @@ job creation fails. -- +[float] +[[ml-categorizationanalyzer]] +==== Categorization Analyzer + +The categorization analyzer specifies how the `categorization_field` will be +interpreted by the categorization process. The syntax is very similar to that +used to define the `analyzer` in the {ref}/analyze.html[Analyze endpoint]. + +The `categorization_analyzer` field can be specified either as a string or as +an object. + +If it is a string it must refer to a +{ref}/analysis-analyzers.html[built-in analyzer] or one added by +another plugin. + +If it is an object it has the following properties: + +`char_filter`:: + (array of strings or objects) One or more + {ref}/analysis-charfilters.html[character filters]. In addition + to the built-in character filters other plugins may provide more. This property + is optional. If not specified then there will be no character filters. If + you are customizing some other aspect of the analyzer and need to achieve + the equivalent of `categorization_filters` (which are not permitted when some + other aspect of the analyzer is customized), add them here as + {ref}/analysis-pattern-replace-charfilter.html[pattern replace character filters]. + +`tokenizer`:: + (string or object) The name or definition of the + {ref}/analysis-tokenizers.html[tokenizer] to use after character + filters have been applied. This property is compulsory if `categorization_analyzer` + is specified as an object. Machine learning provides a tokenizer called `ml_classic` + that tokenizes in the same way as the non-customizable tokenizer in older versions of + the product. If you would like to stick with this but change the character or token + filters then specify `"tokenizer": "ml_classic"` in your `categorization_analyzer`. + +`filter`:: + (array of strings or objects) One or more + {ref}/analysis-tokenfilters.html[token filters]. In addition to the built-in token + filters other plugins may provide more. This property is optional. If not specified + then there will be no token filters. + +If you omit `categorization_analyzer` entirely then the default that will be used is +the one from the following job: + +[source,js] +-------------------------------------------------- +POST _xpack/ml/anomaly_detectors/_validate +{ + "analysis_config" : { + "categorization_analyzer" : { + "tokenizer" : "ml_classic", + "filter" : [ + { "type" : "stop", "stopwords": [ + "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday", + "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun", + "January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December", + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", + "GMT", "UTC" + ] } + ] + }, + "categorization_field_name": "message", + "detectors" :[{ + "function":"count", + "by_field_name": "mlcategory" + }] + }, + "data_description" : { + } +} +-------------------------------------------------- +// CONSOLE + +However, if you specify any part of `categorization_analyzer` then any omitted +sub-properties are _not_ defaulted. + +If you are categorizing non-English messages in a language where words are separated +by spaces you may get better results if you change the day/month words in the stop +token filter to those from your language. If you are categorizing messages in a language +where words are not separated by spaces then you will need to use a different tokenizer +as well in order to get sensible categorization results. + +It is important to be aware that analyzing for categorization of machine generated +log messages is a little different to tokenizing for search. Features that work well +for search, such as stemming, synonym substitution and lowercasing are likely to make +the results of categorization worse. However, in order for drilldown from machine +learning results to work correctly, the tokens that the categorization analyzer +produces need to be sufficiently similar to those produced by the search analyzer +that if you search for the tokens that the categorization analyzer produces you will +find the original document that the field to be categorized came from. + +For more information, see {xpack-ref}/ml-configuring-categories.html[Categorizing Log Messages]. +//<>. + + [float] [[ml-apilimits]] ==== Analysis Limits From 86f612ae3a6c4a23a22d4b246d83e2da0c288801 Mon Sep 17 00:00:00 2001 From: lcawley Date: Mon, 15 Jan 2018 08:29:08 -0800 Subject: [PATCH 05/15] [DOCS] Fixed link to Analyze API Original commit: elastic/x-pack-elasticsearch@e203d839c213052e5e761a533e66c8f45ba2ba25 --- docs/en/rest-api/ml/jobresource.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/rest-api/ml/jobresource.asciidoc b/docs/en/rest-api/ml/jobresource.asciidoc index 2f3026fff08..42a560b2604 100644 --- a/docs/en/rest-api/ml/jobresource.asciidoc +++ b/docs/en/rest-api/ml/jobresource.asciidoc @@ -309,9 +309,9 @@ job creation fails. [[ml-categorizationanalyzer]] ==== Categorization Analyzer -The categorization analyzer specifies how the `categorization_field` will be +The categorization analyzer specifies how the `categorization_field` is interpreted by the categorization process. The syntax is very similar to that -used to define the `analyzer` in the {ref}/analyze.html[Analyze endpoint]. +used to define the `analyzer` in the <>. The `categorization_analyzer` field can be specified either as a string or as an object. From 93b8ef49e9b5a3e75368fe9d23fcea5631ac17d4 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Mon, 15 Jan 2018 18:34:17 +0200 Subject: [PATCH 06/15] Fix CI failure elastic/x-pack-elasticsearch#3568 Bad test design & excessive randomness Original commit: elastic/x-pack-elasticsearch@076d0b246ab95965658810ed796fb9be2724518c --- .../logfile/LoggingAuditTrailFilterTests.java | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java index 8f361a249be..18ab7a89849 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -908,10 +909,7 @@ public class LoggingAuditTrailFilterTests extends ESTestCase { } else { _unfilteredRoles = randomListFromLengthBetween(unfilteredPolicy, 1, unfilteredPolicy.size()); // add roles from other filter policies - final List otherRoles = new ArrayList<>(); - for (int j = 0; j < randomIntBetween(1, 4); j++) { - otherRoles.add(FILTER_MARKER + randomAlphaOfLengthBetween(1, 4)); - } + final List otherRoles = randomNonEmptyListOfFilteredNames("other"); _unfilteredRoles.addAll(randomListFromLengthBetween(otherRoles, 1, otherRoles.size())); settingsBuilder.putList("xpack.security.audit.logfile.events.ignore_filters.otherPolicy.roles", otherRoles); } @@ -1161,7 +1159,7 @@ public class LoggingAuditTrailFilterTests extends ESTestCase { Collections.emptyList()); } } - // filtered roles are a subset of the roles of any policy + // filtered indices are a subset of the indices of any policy final List filterPolicy = randomFrom(allFilteredIndices); final String[] filteredIndices = randomListFromLengthBetween(filterPolicy, 1, filterPolicy.size()).toArray(new String[0]); // unfiltered index sets either have indices distinct from any other in any @@ -1177,10 +1175,7 @@ public class LoggingAuditTrailFilterTests extends ESTestCase { } else { _unfilteredIndices = randomListFromLengthBetween(unfilteredPolicy, 1, unfilteredPolicy.size()); // add indices from other filter policies - final List otherIndices = new ArrayList<>(); - for (int j = 0; j < randomIntBetween(1, 4); j++) { - otherIndices.add(FILTER_MARKER + randomAlphaOfLengthBetween(1, 4)); - } + final List otherIndices = randomNonEmptyListOfFilteredNames("other"); _unfilteredIndices.addAll(randomListFromLengthBetween(otherIndices, 1, otherIndices.size())); settingsBuilder.putList("xpack.security.audit.logfile.events.ignore_filters.otherPolicy.indices", otherIndices); } @@ -1506,10 +1501,10 @@ public class LoggingAuditTrailFilterTests extends ESTestCase { return ans; } - private List randomNonEmptyListOfFilteredNames() { + private List randomNonEmptyListOfFilteredNames(String... namePrefix) { final List filtered = new ArrayList<>(4); for (int i = 0; i < randomIntBetween(1, 4); i++) { - filtered.add(FILTER_MARKER + randomAlphaOfLengthBetween(1, 4)); + filtered.add(FILTER_MARKER + Strings.arrayToCommaDelimitedString(namePrefix) + randomAlphaOfLengthBetween(1, 4)); } return filtered; } From c243d6eb21c54a7afca65b28ed9347f24bee2733 Mon Sep 17 00:00:00 2001 From: lcawley Date: Mon, 15 Jan 2018 08:44:08 -0800 Subject: [PATCH 07/15] [DOCS] Fixed short descriptions for ML APIs Original commit: elastic/x-pack-elasticsearch@75937c0da1875a41542f5d9118d561cff42e524b --- docs/en/rest-api/ml/close-job.asciidoc | 2 +- docs/en/rest-api/ml/delete-calendar-job.asciidoc | 2 +- docs/en/rest-api/ml/delete-calendar.asciidoc | 2 +- docs/en/rest-api/ml/delete-datafeed.asciidoc | 2 +- docs/en/rest-api/ml/delete-job.asciidoc | 2 +- docs/en/rest-api/ml/delete-snapshot.asciidoc | 2 +- docs/en/rest-api/ml/flush-job.asciidoc | 4 ++-- docs/en/rest-api/ml/forecast.asciidoc | 4 ++-- docs/en/rest-api/ml/get-bucket.asciidoc | 5 +++-- docs/en/rest-api/ml/get-calendar.asciidoc | 2 +- docs/en/rest-api/ml/get-category.asciidoc | 3 +-- docs/en/rest-api/ml/get-datafeed-stats.asciidoc | 3 +-- docs/en/rest-api/ml/get-datafeed.asciidoc | 3 +-- docs/en/rest-api/ml/get-influencer.asciidoc | 3 +-- docs/en/rest-api/ml/get-job-stats.asciidoc | 2 +- docs/en/rest-api/ml/get-job.asciidoc | 2 +- docs/en/rest-api/ml/get-overall-buckets.asciidoc | 4 ++-- docs/en/rest-api/ml/get-record.asciidoc | 2 +- docs/en/rest-api/ml/get-snapshot.asciidoc | 2 +- docs/en/rest-api/ml/open-job.asciidoc | 1 + docs/en/rest-api/ml/post-data.asciidoc | 2 +- docs/en/rest-api/ml/preview-datafeed.asciidoc | 8 ++++---- docs/en/rest-api/ml/put-calendar-job.asciidoc | 2 +- docs/en/rest-api/ml/put-calendar.asciidoc | 2 +- docs/en/rest-api/ml/put-datafeed.asciidoc | 2 +- docs/en/rest-api/ml/put-job.asciidoc | 2 +- docs/en/rest-api/ml/revert-snapshot.asciidoc | 16 ++++++++-------- docs/en/rest-api/ml/start-datafeed.asciidoc | 1 + docs/en/rest-api/ml/stop-datafeed.asciidoc | 2 +- docs/en/rest-api/ml/update-datafeed.asciidoc | 2 +- docs/en/rest-api/ml/update-job.asciidoc | 2 +- docs/en/rest-api/ml/update-snapshot.asciidoc | 2 +- docs/en/rest-api/ml/validate-detector.asciidoc | 5 +++-- docs/en/rest-api/ml/validate-job.asciidoc | 5 +++-- 34 files changed, 53 insertions(+), 52 deletions(-) diff --git a/docs/en/rest-api/ml/close-job.asciidoc b/docs/en/rest-api/ml/close-job.asciidoc index e29442da47d..7009692673c 100644 --- a/docs/en/rest-api/ml/close-job.asciidoc +++ b/docs/en/rest-api/ml/close-job.asciidoc @@ -5,7 +5,7 @@ Close Jobs ++++ -The close job API enables you to close one or more jobs. +This API enables you to close one or more jobs. A job can be opened and closed multiple times throughout its lifecycle. A closed job cannot receive data or perform analysis diff --git a/docs/en/rest-api/ml/delete-calendar-job.asciidoc b/docs/en/rest-api/ml/delete-calendar-job.asciidoc index be959092e1d..65ffeb7d32a 100644 --- a/docs/en/rest-api/ml/delete-calendar-job.asciidoc +++ b/docs/en/rest-api/ml/delete-calendar-job.asciidoc @@ -5,7 +5,7 @@ Delete Jobs from Calendar ++++ -The delete jobs from calendar API enables you to remove jobs from a calendar. +This API enables you to remove jobs from a calendar. ==== Request diff --git a/docs/en/rest-api/ml/delete-calendar.asciidoc b/docs/en/rest-api/ml/delete-calendar.asciidoc index fc4ba7c5485..92085769f0e 100644 --- a/docs/en/rest-api/ml/delete-calendar.asciidoc +++ b/docs/en/rest-api/ml/delete-calendar.asciidoc @@ -5,7 +5,7 @@ Delete Calendar ++++ -The delete calendar API enables you to delete a calendar. +This API enables you to delete a calendar. ==== Request diff --git a/docs/en/rest-api/ml/delete-datafeed.asciidoc b/docs/en/rest-api/ml/delete-datafeed.asciidoc index c50e88734ee..eab8c061a2d 100644 --- a/docs/en/rest-api/ml/delete-datafeed.asciidoc +++ b/docs/en/rest-api/ml/delete-datafeed.asciidoc @@ -5,7 +5,7 @@ Delete {dfeeds-cap} ++++ -The delete {dfeed} API enables you to delete an existing {dfeed}. +This API enables you to delete an existing {dfeed}. ==== Request diff --git a/docs/en/rest-api/ml/delete-job.asciidoc b/docs/en/rest-api/ml/delete-job.asciidoc index 1f746e55d9b..e404e9d5f85 100644 --- a/docs/en/rest-api/ml/delete-job.asciidoc +++ b/docs/en/rest-api/ml/delete-job.asciidoc @@ -5,7 +5,7 @@ Delete Jobs ++++ -The delete job API enables you to delete an existing anomaly detection job. +This API enables you to delete an existing anomaly detection job. ==== Request diff --git a/docs/en/rest-api/ml/delete-snapshot.asciidoc b/docs/en/rest-api/ml/delete-snapshot.asciidoc index 16e9969bcac..b63e37a1b45 100644 --- a/docs/en/rest-api/ml/delete-snapshot.asciidoc +++ b/docs/en/rest-api/ml/delete-snapshot.asciidoc @@ -5,7 +5,7 @@ Delete Model Snapshots ++++ -The delete model snapshot API enables you to delete an existing model snapshot. +This API enables you to delete an existing model snapshot. ==== Request diff --git a/docs/en/rest-api/ml/flush-job.asciidoc b/docs/en/rest-api/ml/flush-job.asciidoc index 0a4c055113d..b67a0615af9 100644 --- a/docs/en/rest-api/ml/flush-job.asciidoc +++ b/docs/en/rest-api/ml/flush-job.asciidoc @@ -5,7 +5,7 @@ Flush Jobs ++++ -The flush job API forces any buffered data to be processed by the job. +This API forces any buffered data to be processed by the job. ==== Request @@ -15,7 +15,7 @@ The flush job API forces any buffered data to be processed by the job. ==== Description -The flush job API is only applicable when sending data for analysis using the +The flush jobs API is only applicable when sending data for analysis using the <>. Depending on the content of the buffer, then it might additionally calculate new results. diff --git a/docs/en/rest-api/ml/forecast.asciidoc b/docs/en/rest-api/ml/forecast.asciidoc index 37cab20188a..964d73a4100 100644 --- a/docs/en/rest-api/ml/forecast.asciidoc +++ b/docs/en/rest-api/ml/forecast.asciidoc @@ -5,8 +5,8 @@ Forecast Jobs ++++ -The forecast jobs API uses historical behavior to predict the future behavior of -a time series. +This API uses historical behavior to predict the future behavior of a time +series. ==== Request diff --git a/docs/en/rest-api/ml/get-bucket.asciidoc b/docs/en/rest-api/ml/get-bucket.asciidoc index f62b9d56c3e..9a20d4fc15e 100644 --- a/docs/en/rest-api/ml/get-bucket.asciidoc +++ b/docs/en/rest-api/ml/get-bucket.asciidoc @@ -5,7 +5,7 @@ Get Buckets ++++ -The get bucket API enables you to retrieve job results for one or more buckets. +This API enables you to retrieve job results for one or more buckets. ==== Request @@ -17,7 +17,8 @@ The get bucket API enables you to retrieve job results for one or more buckets. ==== Description -This API presents a chronological view of the records, grouped by bucket. +The get buckets API presents a chronological view of the records, grouped by +bucket. ==== Path Parameters diff --git a/docs/en/rest-api/ml/get-calendar.asciidoc b/docs/en/rest-api/ml/get-calendar.asciidoc index 0a6029633d3..d2936522766 100644 --- a/docs/en/rest-api/ml/get-calendar.asciidoc +++ b/docs/en/rest-api/ml/get-calendar.asciidoc @@ -5,7 +5,7 @@ Get Calendars ++++ -The get calendars API enables you to retrieve configuration information for +This API enables you to retrieve configuration information for calendars. diff --git a/docs/en/rest-api/ml/get-category.asciidoc b/docs/en/rest-api/ml/get-category.asciidoc index a0eae894de0..37d0a95c14c 100644 --- a/docs/en/rest-api/ml/get-category.asciidoc +++ b/docs/en/rest-api/ml/get-category.asciidoc @@ -5,8 +5,7 @@ Get Categories ++++ -The get categories API enables you to retrieve job results for one or more -categories. +This API enables you to retrieve job results for one or more categories. ==== Request diff --git a/docs/en/rest-api/ml/get-datafeed-stats.asciidoc b/docs/en/rest-api/ml/get-datafeed-stats.asciidoc index 4ee799271af..085b8c2df46 100644 --- a/docs/en/rest-api/ml/get-datafeed-stats.asciidoc +++ b/docs/en/rest-api/ml/get-datafeed-stats.asciidoc @@ -5,8 +5,7 @@ Get {dfeed-cap} Statistics ++++ -The get {dfeed} statistics API enables you to retrieve usage information for -{dfeeds}. +This API enables you to retrieve usage information for {dfeeds}. ==== Request diff --git a/docs/en/rest-api/ml/get-datafeed.asciidoc b/docs/en/rest-api/ml/get-datafeed.asciidoc index ac612ae6943..cf31207ba21 100644 --- a/docs/en/rest-api/ml/get-datafeed.asciidoc +++ b/docs/en/rest-api/ml/get-datafeed.asciidoc @@ -5,8 +5,7 @@ Get {dfeeds-cap} ++++ -The get {dfeeds} API enables you to retrieve configuration information for -{dfeeds}. +This API enables you to retrieve configuration information for {dfeeds}. ==== Request diff --git a/docs/en/rest-api/ml/get-influencer.asciidoc b/docs/en/rest-api/ml/get-influencer.asciidoc index 125197373bf..6c49e66e944 100644 --- a/docs/en/rest-api/ml/get-influencer.asciidoc +++ b/docs/en/rest-api/ml/get-influencer.asciidoc @@ -5,8 +5,7 @@ Get Influencers ++++ -The get influencers API enables you to retrieve job results for one or more -influencers. +This API enables you to retrieve job results for one or more influencers. ==== Request diff --git a/docs/en/rest-api/ml/get-job-stats.asciidoc b/docs/en/rest-api/ml/get-job-stats.asciidoc index d9d2273330b..a35e326eae2 100644 --- a/docs/en/rest-api/ml/get-job-stats.asciidoc +++ b/docs/en/rest-api/ml/get-job-stats.asciidoc @@ -5,7 +5,7 @@ Get Job Statistics ++++ -The get jobs API enables you to retrieve usage information for jobs. +This API enables you to retrieve usage information for jobs. ==== Request diff --git a/docs/en/rest-api/ml/get-job.asciidoc b/docs/en/rest-api/ml/get-job.asciidoc index 7a8f4c127b5..0818983406e 100644 --- a/docs/en/rest-api/ml/get-job.asciidoc +++ b/docs/en/rest-api/ml/get-job.asciidoc @@ -5,7 +5,7 @@ Get Jobs ++++ -The get jobs API enables you to retrieve configuration information for jobs. +This API enables you to retrieve configuration information for jobs. ==== Request diff --git a/docs/en/rest-api/ml/get-overall-buckets.asciidoc b/docs/en/rest-api/ml/get-overall-buckets.asciidoc index eee6cec9c04..d0e8c1f214b 100644 --- a/docs/en/rest-api/ml/get-overall-buckets.asciidoc +++ b/docs/en/rest-api/ml/get-overall-buckets.asciidoc @@ -5,8 +5,8 @@ Get Overall Buckets ++++ -This API enables you to retrieve overall bucket results -that summarize the bucket results of multiple jobs. +This API enables you to retrieve overall bucket results that summarize the +bucket results of multiple jobs. ==== Request diff --git a/docs/en/rest-api/ml/get-record.asciidoc b/docs/en/rest-api/ml/get-record.asciidoc index abe00f6547f..6cd222027e6 100644 --- a/docs/en/rest-api/ml/get-record.asciidoc +++ b/docs/en/rest-api/ml/get-record.asciidoc @@ -5,7 +5,7 @@ Get Records ++++ -The get records API enables you to retrieve anomaly records for a job. +This API enables you to retrieve anomaly records for a job. ==== Request diff --git a/docs/en/rest-api/ml/get-snapshot.asciidoc b/docs/en/rest-api/ml/get-snapshot.asciidoc index 53ce0a617a3..7ac2a3d765f 100644 --- a/docs/en/rest-api/ml/get-snapshot.asciidoc +++ b/docs/en/rest-api/ml/get-snapshot.asciidoc @@ -5,7 +5,7 @@ Get Model Snapshots ++++ -The get model snapshots API enables you to retrieve information about model snapshots. +This API enables you to retrieve information about model snapshots. ==== Request diff --git a/docs/en/rest-api/ml/open-job.asciidoc b/docs/en/rest-api/ml/open-job.asciidoc index a1dec2d684c..37d201ed226 100644 --- a/docs/en/rest-api/ml/open-job.asciidoc +++ b/docs/en/rest-api/ml/open-job.asciidoc @@ -5,6 +5,7 @@ Open Jobs ++++ +This API enables you to open one or more jobs. A job must be opened in order for it to be ready to receive and analyze data. A job can be opened and closed multiple times throughout its lifecycle. diff --git a/docs/en/rest-api/ml/post-data.asciidoc b/docs/en/rest-api/ml/post-data.asciidoc index 5daeb8b2f67..ec20be5dadb 100644 --- a/docs/en/rest-api/ml/post-data.asciidoc +++ b/docs/en/rest-api/ml/post-data.asciidoc @@ -5,7 +5,7 @@ Post Data to Jobs ++++ -The post data API enables you to send data to an anomaly detection job for analysis. +This API enables you to send data to an anomaly detection job for analysis. ==== Request diff --git a/docs/en/rest-api/ml/preview-datafeed.asciidoc b/docs/en/rest-api/ml/preview-datafeed.asciidoc index dfb402efa92..5f3bc5054e3 100644 --- a/docs/en/rest-api/ml/preview-datafeed.asciidoc +++ b/docs/en/rest-api/ml/preview-datafeed.asciidoc @@ -5,7 +5,7 @@ Preview {dfeeds-cap} ++++ -The preview {dfeed} API enables you to preview a {dfeed}. +This API enables you to preview a {dfeed}. ==== Request @@ -15,9 +15,9 @@ The preview {dfeed} API enables you to preview a {dfeed}. ==== Description -The API returns the first "page" of results from the `search` that is created -by using the current {dfeed} settings. This preview shows the structure of -the data that will be passed to the anomaly detection engine. +The preview {dfeeds} API returns the first "page" of results from the `search` +that is created by using the current {dfeed} settings. This preview shows the +structure of the data that will be passed to the anomaly detection engine. ==== Path Parameters diff --git a/docs/en/rest-api/ml/put-calendar-job.asciidoc b/docs/en/rest-api/ml/put-calendar-job.asciidoc index b7c550e1959..d49c4c62e79 100644 --- a/docs/en/rest-api/ml/put-calendar-job.asciidoc +++ b/docs/en/rest-api/ml/put-calendar-job.asciidoc @@ -5,7 +5,7 @@ Add Jobs to Calendar ++++ -The add jobs to calendar API enables you to add jobs to a calendar. +This API enables you to add jobs to a calendar. ==== Request diff --git a/docs/en/rest-api/ml/put-calendar.asciidoc b/docs/en/rest-api/ml/put-calendar.asciidoc index 5f8714b1d58..2598c35007f 100644 --- a/docs/en/rest-api/ml/put-calendar.asciidoc +++ b/docs/en/rest-api/ml/put-calendar.asciidoc @@ -5,7 +5,7 @@ Create Calendar ++++ -The create calendar API enables you to instantiate a calendar. +This API enables you to instantiate a calendar. ==== Request diff --git a/docs/en/rest-api/ml/put-datafeed.asciidoc b/docs/en/rest-api/ml/put-datafeed.asciidoc index 2e367273690..00d678d3091 100644 --- a/docs/en/rest-api/ml/put-datafeed.asciidoc +++ b/docs/en/rest-api/ml/put-datafeed.asciidoc @@ -5,7 +5,7 @@ Create {dfeeds-cap} ++++ -The create {dfeed} API enables you to instantiate a {dfeed}. +This API enables you to instantiate a {dfeed}. ==== Request diff --git a/docs/en/rest-api/ml/put-job.asciidoc b/docs/en/rest-api/ml/put-job.asciidoc index 8356b071dd1..975b87c6f1e 100644 --- a/docs/en/rest-api/ml/put-job.asciidoc +++ b/docs/en/rest-api/ml/put-job.asciidoc @@ -5,7 +5,7 @@ Create Jobs ++++ -The create job API enables you to instantiate a job. +This API enables you to instantiate a job. ==== Request diff --git a/docs/en/rest-api/ml/revert-snapshot.asciidoc b/docs/en/rest-api/ml/revert-snapshot.asciidoc index a835c0f4db3..30e93976543 100644 --- a/docs/en/rest-api/ml/revert-snapshot.asciidoc +++ b/docs/en/rest-api/ml/revert-snapshot.asciidoc @@ -5,7 +5,7 @@ Revert Model Snapshots ++++ -The revert model snapshot API enables you to revert to a specific snapshot. +This API enables you to revert to a specific snapshot. ==== Request @@ -14,13 +14,13 @@ The revert model snapshot API enables you to revert to a specific snapshot. ==== Description -The {ml} feature in {xpack} reacts quickly to anomalous input, learning new behaviors in data. -Highly anomalous input increases the variance in the models whilst the system learns -whether this is a new step-change in behavior or a one-off event. In the case -where this anomalous input is known to be a one-off, then it might be appropriate -to reset the model state to a time before this event. For example, you might -consider reverting to a saved snapshot after Black Friday -or a critical system failure. +The {ml} feature in {xpack} reacts quickly to anomalous input, learning new +behaviors in data. Highly anomalous input increases the variance in the models +whilst the system learns whether this is a new step-change in behavior or a +one-off event. In the case where this anomalous input is known to be a one-off, +then it might be appropriate to reset the model state to a time before this +event. For example, you might consider reverting to a saved snapshot after Black +Friday or a critical system failure. //// To revert to a saved snapshot, you must follow this sequence: diff --git a/docs/en/rest-api/ml/start-datafeed.asciidoc b/docs/en/rest-api/ml/start-datafeed.asciidoc index 801984deac0..865ca4ae997 100644 --- a/docs/en/rest-api/ml/start-datafeed.asciidoc +++ b/docs/en/rest-api/ml/start-datafeed.asciidoc @@ -5,6 +5,7 @@ Start {dfeeds-cap} ++++ +This API enables you to start one or more {dfeeds}. A {dfeed} must be started in order to retrieve data from {es}. A {dfeed} can be started and stopped multiple times throughout its lifecycle. diff --git a/docs/en/rest-api/ml/stop-datafeed.asciidoc b/docs/en/rest-api/ml/stop-datafeed.asciidoc index ca616b67404..3511c9362c3 100644 --- a/docs/en/rest-api/ml/stop-datafeed.asciidoc +++ b/docs/en/rest-api/ml/stop-datafeed.asciidoc @@ -5,7 +5,7 @@ Stop {dfeeds-cap} ++++ -The stop {dfeeds} API enables you to stop one or more {dfeeds}. +This API enables you to stop one or more {dfeeds}. A {dfeed} that is stopped ceases to retrieve data from {es}. A {dfeed} can be started and stopped multiple times throughout its lifecycle. diff --git a/docs/en/rest-api/ml/update-datafeed.asciidoc b/docs/en/rest-api/ml/update-datafeed.asciidoc index 560f3c82ca2..4643236c170 100644 --- a/docs/en/rest-api/ml/update-datafeed.asciidoc +++ b/docs/en/rest-api/ml/update-datafeed.asciidoc @@ -5,7 +5,7 @@ Update {dfeeds-cap} ++++ -The update {dfeed} API enables you to update certain properties of a {dfeed}. +This API enables you to update certain properties of a {dfeed}. ==== Request diff --git a/docs/en/rest-api/ml/update-job.asciidoc b/docs/en/rest-api/ml/update-job.asciidoc index 25640b32c82..f8bc3c10376 100644 --- a/docs/en/rest-api/ml/update-job.asciidoc +++ b/docs/en/rest-api/ml/update-job.asciidoc @@ -5,7 +5,7 @@ Update Jobs ++++ -The update job API enables you to update certain properties of a job. +This API enables you to update certain properties of a job. ==== Request diff --git a/docs/en/rest-api/ml/update-snapshot.asciidoc b/docs/en/rest-api/ml/update-snapshot.asciidoc index 30a2c0c5143..74a684619c4 100644 --- a/docs/en/rest-api/ml/update-snapshot.asciidoc +++ b/docs/en/rest-api/ml/update-snapshot.asciidoc @@ -5,7 +5,7 @@ Update Model Snapshots ++++ -The update model snapshot API enables you to update certain properties of a snapshot. +This API enables you to update certain properties of a snapshot. ==== Request diff --git a/docs/en/rest-api/ml/validate-detector.asciidoc b/docs/en/rest-api/ml/validate-detector.asciidoc index 60c0c7f0646..6fc5fea6fbb 100644 --- a/docs/en/rest-api/ml/validate-detector.asciidoc +++ b/docs/en/rest-api/ml/validate-detector.asciidoc @@ -5,7 +5,7 @@ Validate Detectors ++++ -The validate detectors API validates detector configuration information. +This API validates detector configuration information. ==== Request @@ -13,7 +13,8 @@ The validate detectors API validates detector configuration information. ==== Description -This API enables you validate the detector configuration before you create a job. +The validate detectors API enables you validate the detector configuration +before you create a job. ==== Request Body diff --git a/docs/en/rest-api/ml/validate-job.asciidoc b/docs/en/rest-api/ml/validate-job.asciidoc index 18a6e96da32..b206734bc03 100644 --- a/docs/en/rest-api/ml/validate-job.asciidoc +++ b/docs/en/rest-api/ml/validate-job.asciidoc @@ -5,7 +5,7 @@ Validate Jobs ++++ -The validate jobs API validates job configuration information. +This API validates job configuration information. ==== Request @@ -13,7 +13,8 @@ The validate jobs API validates job configuration information. ==== Description -This API enables you validate the job configuration before you create the job. +The validate jobs API enables you validate the job configuration before you +create the job. ==== Request Body From 5eef187b19244fa0af7c1d9378c8ad88082d4365 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Mon, 15 Jan 2018 09:59:03 -0700 Subject: [PATCH 08/15] Modify x-pack to support elasticsearch-core jar (elastic/x-pack-elasticsearch#3544) This is related to elastic/elasticsearch#27933. It modifies x-pack to be compatible with the new elasticsearch-core jar. Original commit: elastic/x-pack-elasticsearch@42227b603493f0e8a560cadcfa1b35fe1d540a1b --- .../watcher/transform/chain/ChainTransformFactory.java | 4 ++-- .../xpack/ml/MlDailyMaintenanceService.java | 2 -- .../autodetect/output/AutoDetectResultProcessor.java | 1 - .../elasticsearch/xpack/security/SecurityContext.java | 4 ++-- .../xpack/security/authc/ExpiredTokenRemover.java | 4 ++-- .../elasticsearch/xpack/security/authc/RealmConfig.java | 4 ++-- .../authc/esnative/ESNativeRealmMigrateTool.java | 8 ++++---- .../accesscontrol/SecurityIndexSearcherWrapper.java | 4 ++-- .../xpack/security/transport/filter/IPFilter.java | 4 ++-- .../elasticsearch/xpack/ssl/RestrictedTrustManager.java | 4 ++-- .../xpack/watcher/actions/email/EmailActionFactory.java | 4 ++-- .../watcher/actions/hipchat/HipChatActionFactory.java | 4 ++-- .../xpack/watcher/actions/index/IndexActionFactory.java | 4 ++-- .../xpack/watcher/actions/jira/JiraActionFactory.java | 4 ++-- .../watcher/actions/logging/ExecutableLoggingAction.java | 4 ++-- .../watcher/actions/logging/LoggingActionFactory.java | 4 ++-- .../actions/pagerduty/PagerDutyActionFactory.java | 4 ++-- .../xpack/watcher/actions/slack/SlackActionFactory.java | 4 ++-- .../watcher/actions/webhook/WebhookActionFactory.java | 4 ++-- .../watcher/execution/AsyncTriggerEventConsumer.java | 4 ++-- .../watcher/execution/SyncTriggerEventConsumer.java | 4 ++-- .../xpack/watcher/input/chain/ChainInputFactory.java | 4 ++-- .../xpack/watcher/input/http/HttpInputFactory.java | 4 ++-- .../xpack/watcher/input/none/NoneInputFactory.java | 4 ++-- .../xpack/watcher/input/search/SearchInputFactory.java | 4 ++-- .../xpack/watcher/input/simple/SimpleInputFactory.java | 4 ++-- .../watcher/input/transform/TransformInputFactory.java | 4 ++-- .../watcher/transform/script/ScriptTransformFactory.java | 4 ++-- .../watcher/transform/search/SearchTransformFactory.java | 4 ++-- .../job/process/logging/CppLogMessageHandlerTests.java | 9 +++++---- .../xpack/security/audit/logfile/CapturingLogger.java | 6 +++--- .../xpack/watcher/test/TimeWarpedWatcher.java | 4 ++-- .../test/bench/WatcherExecutorServiceBenchmark.java | 4 ++-- .../xpack/watcher/trigger/ScheduleTriggerEngineMock.java | 4 ++-- 34 files changed, 70 insertions(+), 72 deletions(-) diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/watcher/transform/chain/ChainTransformFactory.java b/plugin/core/src/main/java/org/elasticsearch/xpack/watcher/transform/chain/ChainTransformFactory.java index 0ec022302d0..979499c6cad 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/watcher/transform/chain/ChainTransformFactory.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/watcher/transform/chain/ChainTransformFactory.java @@ -5,7 +5,7 @@ */ package org.elasticsearch.xpack.watcher.transform.chain; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.watcher.transform.ExecutableTransform; @@ -21,7 +21,7 @@ public final class ChainTransformFactory extends TransformFactory Loggers.removeAppender(logger, a)); - Loggers.addAppender(logger, appender); + loggerConfig.getAppenders().forEach((s, a) -> ServerLoggers.removeAppender(logger, a)); + ServerLoggers.addAppender(logger, appender); return logger; } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java b/plugin/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java index 375d414c694..56d7a304ace 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java @@ -30,7 +30,7 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.logging.LoggerMessageFormat; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -98,7 +98,7 @@ public class SecurityIndexSearcherWrapper extends IndexSearcherWrapper { BitsetFilterCache bitsetFilterCache, ThreadContext threadContext, XPackLicenseState licenseState, ScriptService scriptService) { this.scriptService = scriptService; - this.logger = Loggers.getLogger(getClass(), indexSettings.getSettings()); + this.logger = ServerLoggers.getLogger(getClass(), indexSettings.getSettings()); this.queryShardContextProvider = queryShardContextProvider; this.bitsetFilterCache = bitsetFilterCache; this.threadContext = threadContext; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/transport/filter/IPFilter.java b/plugin/src/main/java/org/elasticsearch/xpack/security/transport/filter/IPFilter.java index 719d052889e..07d72353d8c 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/transport/filter/IPFilter.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/transport/filter/IPFilter.java @@ -10,7 +10,7 @@ import io.netty.handler.ipfilter.IpFilterRuleType; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SetOnce; import org.elasticsearch.common.collect.MapBuilder; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; @@ -117,7 +117,7 @@ public class IPFilter { public IPFilter(final Settings settings, AuditTrailService auditTrail, ClusterSettings clusterSettings, XPackLicenseState licenseState) { - this.logger = Loggers.getLogger(getClass(), settings); + this.logger = ServerLoggers.getLogger(getClass(), settings); this.auditTrail = auditTrail; this.licenseState = licenseState; this.alwaysAllowBoundAddresses = ALLOW_BOUND_ADDRESSES_SETTING.get(settings); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ssl/RestrictedTrustManager.java b/plugin/src/main/java/org/elasticsearch/xpack/ssl/RestrictedTrustManager.java index 76989cd594d..e6921f5ec87 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ssl/RestrictedTrustManager.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ssl/RestrictedTrustManager.java @@ -28,7 +28,7 @@ import org.bouncycastle.asn1.ASN1Sequence; import org.bouncycastle.asn1.ASN1String; import org.bouncycastle.asn1.ASN1TaggedObject; import org.bouncycastle.asn1.DERTaggedObject; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.settings.Settings; /** @@ -46,7 +46,7 @@ public final class RestrictedTrustManager extends X509ExtendedTrustManager { private final int SAN_CODE_OTHERNAME = 0; public RestrictedTrustManager(Settings settings, X509ExtendedTrustManager delegate, CertificateTrustRestrictions restrictions) { - this.logger = Loggers.getLogger(getClass(), settings); + this.logger = ServerLoggers.getLogger(getClass(), settings); this.delegate = delegate; this.trustRestrictions = restrictions; logger.debug("Configured with trust restrictions: [{}]", restrictions); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/email/EmailActionFactory.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/email/EmailActionFactory.java index 7f664c3ac1d..178840a5686 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/email/EmailActionFactory.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/email/EmailActionFactory.java @@ -5,7 +5,7 @@ */ package org.elasticsearch.xpack.watcher.actions.email; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; @@ -25,7 +25,7 @@ public class EmailActionFactory extends ActionFactory { public EmailActionFactory(Settings settings, EmailService emailService, TextTemplateEngine templateEngine, EmailAttachmentsParser emailAttachmentsParser) { - super(Loggers.getLogger(ExecutableEmailAction.class, settings)); + super(ServerLoggers.getLogger(ExecutableEmailAction.class, settings)); this.emailService = emailService; this.templateEngine = templateEngine; this.htmlSanitizer = new HtmlSanitizer(settings); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/hipchat/HipChatActionFactory.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/hipchat/HipChatActionFactory.java index a94f4cc1339..5d6d02fd558 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/hipchat/HipChatActionFactory.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/hipchat/HipChatActionFactory.java @@ -5,7 +5,7 @@ */ package org.elasticsearch.xpack.watcher.actions.hipchat; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; @@ -21,7 +21,7 @@ public class HipChatActionFactory extends ActionFactory { private final HipChatService hipchatService; public HipChatActionFactory(Settings settings, TextTemplateEngine templateEngine, HipChatService hipchatService) { - super(Loggers.getLogger(ExecutableHipChatAction.class, settings)); + super(ServerLoggers.getLogger(ExecutableHipChatAction.class, settings)); this.templateEngine = templateEngine; this.hipchatService = hipchatService; } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/index/IndexActionFactory.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/index/IndexActionFactory.java index 94e3ec4be4b..05f39f108e7 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/index/IndexActionFactory.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/index/IndexActionFactory.java @@ -6,7 +6,7 @@ package org.elasticsearch.xpack.watcher.actions.index; import org.elasticsearch.client.Client; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; @@ -21,7 +21,7 @@ public class IndexActionFactory extends ActionFactory { private final TimeValue bulkDefaultTimeout; public IndexActionFactory(Settings settings, Client client) { - super(Loggers.getLogger(IndexActionFactory.class, settings)); + super(ServerLoggers.getLogger(IndexActionFactory.class, settings)); this.client = client; this.indexDefaultTimeout = settings.getAsTime("xpack.watcher.actions.index.default_timeout", TimeValue.timeValueSeconds(30)); this.bulkDefaultTimeout = settings.getAsTime("xpack.watcher.actions.bulk.default_timeout", TimeValue.timeValueMinutes(1)); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/jira/JiraActionFactory.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/jira/JiraActionFactory.java index 1f74e15e584..29e0d09840e 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/jira/JiraActionFactory.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/jira/JiraActionFactory.java @@ -5,7 +5,7 @@ */ package org.elasticsearch.xpack.watcher.actions.jira; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; @@ -20,7 +20,7 @@ public class JiraActionFactory extends ActionFactory { private final JiraService jiraService; public JiraActionFactory(Settings settings, TextTemplateEngine templateEngine, JiraService jiraService) { - super(Loggers.getLogger(ExecutableJiraAction.class, settings)); + super(ServerLoggers.getLogger(ExecutableJiraAction.class, settings)); this.templateEngine = templateEngine; this.jiraService = jiraService; } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/logging/ExecutableLoggingAction.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/logging/ExecutableLoggingAction.java index d4a6ec43fa4..32b5f1a7220 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/logging/ExecutableLoggingAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/logging/ExecutableLoggingAction.java @@ -6,7 +6,7 @@ package org.elasticsearch.xpack.watcher.actions.logging; import org.apache.logging.log4j.Logger; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; import org.elasticsearch.xpack.watcher.actions.Action; @@ -24,7 +24,7 @@ public class ExecutableLoggingAction extends ExecutableAction { public ExecutableLoggingAction(LoggingAction action, Logger logger, Settings settings, TextTemplateEngine templateEngine) { super(action, logger); - this.textLogger = action.category != null ? Loggers.getLogger(action.category, settings) : logger; + this.textLogger = action.category != null ? ServerLoggers.getLogger(action.category, settings) : logger; this.templateEngine = templateEngine; } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/logging/LoggingActionFactory.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/logging/LoggingActionFactory.java index 97b7035dbc6..a17fc7dfa14 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/logging/LoggingActionFactory.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/logging/LoggingActionFactory.java @@ -5,7 +5,7 @@ */ package org.elasticsearch.xpack.watcher.actions.logging; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; @@ -19,7 +19,7 @@ public class LoggingActionFactory extends ActionFactory { private final TextTemplateEngine templateEngine; public LoggingActionFactory(Settings settings, TextTemplateEngine templateEngine) { - super(Loggers.getLogger(ExecutableLoggingAction.class, settings)); + super(ServerLoggers.getLogger(ExecutableLoggingAction.class, settings)); this.settings = settings; this.templateEngine = templateEngine; } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/pagerduty/PagerDutyActionFactory.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/pagerduty/PagerDutyActionFactory.java index 02982082ed7..4dd634028bc 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/pagerduty/PagerDutyActionFactory.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/pagerduty/PagerDutyActionFactory.java @@ -5,7 +5,7 @@ */ package org.elasticsearch.xpack.watcher.actions.pagerduty; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; @@ -20,7 +20,7 @@ public class PagerDutyActionFactory extends ActionFactory { private final PagerDutyService pagerDutyService; public PagerDutyActionFactory(Settings settings, TextTemplateEngine templateEngine, PagerDutyService pagerDutyService) { - super(Loggers.getLogger(ExecutablePagerDutyAction.class, settings)); + super(ServerLoggers.getLogger(ExecutablePagerDutyAction.class, settings)); this.templateEngine = templateEngine; this.pagerDutyService = pagerDutyService; } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/slack/SlackActionFactory.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/slack/SlackActionFactory.java index d455921a933..bbd443bee8f 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/slack/SlackActionFactory.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/slack/SlackActionFactory.java @@ -5,7 +5,7 @@ */ package org.elasticsearch.xpack.watcher.actions.slack; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; @@ -19,7 +19,7 @@ public class SlackActionFactory extends ActionFactory { private final SlackService slackService; public SlackActionFactory(Settings settings, TextTemplateEngine templateEngine, SlackService slackService) { - super(Loggers.getLogger(ExecutableSlackAction.class, settings)); + super(ServerLoggers.getLogger(ExecutableSlackAction.class, settings)); this.templateEngine = templateEngine; this.slackService = slackService; } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionFactory.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionFactory.java index c03080ff6e8..0b53b37fcbc 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionFactory.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionFactory.java @@ -5,7 +5,7 @@ */ package org.elasticsearch.xpack.watcher.actions.webhook; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; @@ -24,7 +24,7 @@ public class WebhookActionFactory extends ActionFactory { public WebhookActionFactory(Settings settings, HttpClient httpClient, HttpRequestTemplate.Parser requestTemplateParser, TextTemplateEngine templateEngine) { - super(Loggers.getLogger(ExecutableWebhookAction.class, settings)); + super(ServerLoggers.getLogger(ExecutableWebhookAction.class, settings)); this.httpClient = httpClient; this.requestTemplateParser = requestTemplateParser; this.templateEngine = templateEngine; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/execution/AsyncTriggerEventConsumer.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/execution/AsyncTriggerEventConsumer.java index 7e637185600..bc01e52a22f 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/execution/AsyncTriggerEventConsumer.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/execution/AsyncTriggerEventConsumer.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.watcher.execution; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.util.Supplier; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.watcher.trigger.TriggerEvent; @@ -22,7 +22,7 @@ public class AsyncTriggerEventConsumer implements Consumer private final Logger logger; public SyncTriggerEventConsumer(Settings settings, ExecutionService executionService) { - this.logger = Loggers.getLogger(SyncTriggerEventConsumer.class, settings); + this.logger = ServerLoggers.getLogger(SyncTriggerEventConsumer.class, settings); this.executionService = executionService; } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/input/chain/ChainInputFactory.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/input/chain/ChainInputFactory.java index f5ce4f2a665..792bb233d47 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/input/chain/ChainInputFactory.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/input/chain/ChainInputFactory.java @@ -6,7 +6,7 @@ package org.elasticsearch.xpack.watcher.input.chain; import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.watcher.input.ExecutableInput; @@ -23,7 +23,7 @@ public class ChainInputFactory extends InputFactory { public NoneInputFactory(Settings settings) { - super(Loggers.getLogger(ExecutableNoneInput.class, settings)); + super(ServerLoggers.getLogger(ExecutableNoneInput.class, settings)); } @Override diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java index 2342c3e6ca6..74c55de298e 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/input/search/SearchInputFactory.java @@ -6,7 +6,7 @@ package org.elasticsearch.xpack.watcher.input.search; import org.elasticsearch.client.Client; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -25,7 +25,7 @@ public class SearchInputFactory extends InputFactory { public SimpleInputFactory(Settings settings) { - super(Loggers.getLogger(ExecutableSimpleInput.class, settings)); + super(ServerLoggers.getLogger(ExecutableSimpleInput.class, settings)); } @Override diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/input/transform/TransformInputFactory.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/input/transform/TransformInputFactory.java index 7aa587b3900..9e728e8704a 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/input/transform/TransformInputFactory.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/input/transform/TransformInputFactory.java @@ -5,7 +5,7 @@ */ package org.elasticsearch.xpack.watcher.input.transform; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.logging.ServerLoggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParserUtils; @@ -32,7 +32,7 @@ public final class TransformInputFactory extends InputFactory Date: Mon, 15 Jan 2018 18:05:55 +0100 Subject: [PATCH 09/15] Tests: Add noop test to prevent test failures Follow up for elastic/x-pack-elasticsearch@a15338415ac1bd06fdebad1bf93c5e2fec45de5c Original commit: elastic/x-pack-elasticsearch@c0a46a5977fe0dd28a955c8ab3760f0861b9394a --- .../smoketest/PreventFailingBuildIT.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/PreventFailingBuildIT.java diff --git a/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/PreventFailingBuildIT.java b/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/PreventFailingBuildIT.java new file mode 100644 index 00000000000..2c2cdd044aa --- /dev/null +++ b/qa/smoke-test-monitoring-with-watcher/src/test/java/org/elasticsearch/smoketest/PreventFailingBuildIT.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.smoketest; + +import org.elasticsearch.test.ESIntegTestCase; + +public class PreventFailingBuildIT extends ESIntegTestCase { + + public void testSoThatTestsDoNotFail() { + // Noop + + // This is required because if tests are not enable no + // tests will be run in the entire project and all tests will fail. + } +} From aff9a4a2baa6c1a28b7c528a2993fb9e24ea51fb Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Mon, 15 Jan 2018 18:07:48 +0000 Subject: [PATCH 10/15] [ML] Autodetect should receive events from the earliest valid timestamp (elastic/x-pack-elasticsearch#3570) When events are searched to be passed to the autodetect process, they are currently calculated based on the latest record timestamp, when a job opens, and `now` when the process is updated. This commit changes both to be consistent and based on the earliest valid timestamp for the job. The earliest valid timestamp is the latest record timestamp minus the job latency. Relates elastic/x-pack-elasticsearch#3016 Original commit: elastic/x-pack-elasticsearch@7f882ea0531ccb2a965c9d5e98cfab6cd199c56f --- .../xpack/ml/job/config/Job.java | 18 +++++++++++++ .../xpack/ml/job/persistence/JobProvider.java | 5 +--- .../ScheduledEventsQueryBuilder.java | 5 ++++ .../TransportPostCalendarEventsAction.java | 3 +-- .../ml/action/TransportPutCalendarAction.java | 6 +---- .../autodetect/AutodetectProcessManager.java | 4 +-- .../xpack/ml/job/config/JobTests.java | 25 +++++++++++++++++++ 7 files changed, 53 insertions(+), 13 deletions(-) diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/ml/job/config/Job.java b/plugin/core/src/main/java/org/elasticsearch/xpack/ml/job/config/Job.java index 2e90be13d10..5ee53a0cef7 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/ml/job/config/Job.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/ml/job/config/Job.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.xpack.ml.MlParserType; import org.elasticsearch.xpack.ml.job.messages.Messages; import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndexFields; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.utils.MlStrings; import org.elasticsearch.xpack.ml.utils.time.TimeUtils; @@ -450,6 +451,23 @@ public class Job extends AbstractDiffable implements Writeable, ToXContentO + PROCESS_MEMORY_OVERHEAD.getBytes(); } + /** + * Returns the timestamp before which data is not accepted by the job. + * This is the latest record timestamp minus the job latency. + * @param dataCounts the job data counts + * @return the timestamp before which data is not accepted by the job + */ + public long earliestValidTimestamp(DataCounts dataCounts) { + long currentTime = 0; + Date latestRecordTimestamp = dataCounts.getLatestRecordTimeStamp(); + if (latestRecordTimestamp != null) { + TimeValue latency = analysisConfig.getLatency(); + long latencyMillis = latency == null ? 0 : latency.millis(); + currentTime = latestRecordTimestamp.getTime() - latencyMillis; + } + return currentTime; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(jobId); diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java b/plugin/core/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java index f81e37c99e2..d9617038ef0 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java @@ -369,10 +369,7 @@ public class JobProvider { ActionListener getScheduledEventsListener = ActionListener.wrap( paramsBuilder -> { ScheduledEventsQueryBuilder scheduledEventsQueryBuilder = new ScheduledEventsQueryBuilder(); - Date lastestRecordTime = paramsBuilder.getDataCounts().getLatestRecordTimeStamp(); - if (lastestRecordTime != null) { - scheduledEventsQueryBuilder.start(Long.toString(lastestRecordTime.getTime())); - } + scheduledEventsQueryBuilder.start(job.earliestValidTimestamp(paramsBuilder.getDataCounts())); scheduledEventsForJob(jobId, job.getGroups(), scheduledEventsQueryBuilder, ActionListener.wrap( events -> { paramsBuilder.setScheduledEvents(events.results()); diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ScheduledEventsQueryBuilder.java b/plugin/core/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ScheduledEventsQueryBuilder.java index 954c52f0156..5fbc9433941 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ScheduledEventsQueryBuilder.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ScheduledEventsQueryBuilder.java @@ -41,6 +41,11 @@ public class ScheduledEventsQueryBuilder { return this; } + public ScheduledEventsQueryBuilder start(long start) { + this.start = Long.toString(start); + return this; + } + public ScheduledEventsQueryBuilder end(String end) { this.end = end; return this; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java index 6f119639802..8239db20422 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/TransportPostCalendarEventsAction.java @@ -88,8 +88,7 @@ public class TransportPostCalendarEventsAction extends HandledTransportAction() { @Override public void onResponse(IndexResponse indexResponse) { - jobManager.updateProcessOnCalendarChanged(calendar.getJobIds()); listener.onResponse(new PutCalendarAction.Response(calendar)); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java index 9e946763a18..fecc3448715 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -60,7 +60,6 @@ import java.io.IOException; import java.io.InputStream; import java.time.Duration; import java.time.ZonedDateTime; -import java.util.Collections; import java.util.Date; import java.util.Iterator; import java.util.List; @@ -280,7 +279,8 @@ public class AutodetectProcessManager extends AbstractComponent { if (updateParams.isUpdateScheduledEvents()) { Job job = jobManager.getJobOrThrowIfUnknown(jobTask.getJobId()); - ScheduledEventsQueryBuilder query = new ScheduledEventsQueryBuilder().start(Long.toString(new Date().getTime())); + DataCounts dataCounts = getStatistics(jobTask).get().v1(); + ScheduledEventsQueryBuilder query = new ScheduledEventsQueryBuilder().start(job.earliestValidTimestamp(dataCounts)); jobProvider.scheduledEventsForJob(jobTask.getJobId(), job.getGroups(), query, eventsListener); } else { eventsListener.onResponse(null); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTests.java index 240de975ad8..4db9192e53e 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ml.MachineLearningClientActionPlugin; import org.elasticsearch.xpack.ml.job.messages.Messages; import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndexFields; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; import java.io.IOException; import java.util.ArrayList; @@ -551,6 +552,30 @@ public class JobTests extends AbstractSerializingTestCase { builder.build().estimateMemoryFootprint()); } + public void testEarliestValidTimestamp_GivenEmptyDataCounts() { + assertThat(createRandomizedJob().earliestValidTimestamp(new DataCounts("foo")), equalTo(0L)); + } + + public void testEarliestValidTimestamp_GivenDataCountsAndZeroLatency() { + Job.Builder builder = buildJobBuilder("foo"); + DataCounts dataCounts = new DataCounts(builder.getId()); + dataCounts.setLatestRecordTimeStamp(new Date(123456789L)); + + assertThat(builder.build().earliestValidTimestamp(dataCounts), equalTo(123456789L)); + } + + public void testEarliestValidTimestamp_GivenDataCountsAndLatency() { + Job.Builder builder = buildJobBuilder("foo"); + AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(builder.build().getAnalysisConfig()); + analysisConfig.setLatency(TimeValue.timeValueMillis(1000L)); + builder.setAnalysisConfig(analysisConfig); + + DataCounts dataCounts = new DataCounts(builder.getId()); + dataCounts.setLatestRecordTimeStamp(new Date(123456789L)); + + assertThat(builder.build().earliestValidTimestamp(dataCounts), equalTo(123455789L)); + } + public static Job.Builder buildJobBuilder(String id, Date date) { Job.Builder builder = new Job.Builder(id); builder.setCreateTime(date); From 6c6dcafd0e8ab7ada5fb21d74839e5102697b4fe Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Mon, 15 Jan 2018 18:13:50 +0000 Subject: [PATCH 11/15] [ML] Improve error message when post-calendar-events-request is empty (elastic/x-pack-elasticsearch#3572) Relates elastic/x-pack-elasticsearch#3016 Original commit: elastic/x-pack-elasticsearch@3d25ed47232a7a1a0310c4af09adf3fc1efa5b24 --- .../xpack/ml/action/PostCalendarEventsAction.java | 4 ++++ .../rest-api-spec/test/ml/calendar_crud.yml | 12 ++++++++++++ qa/smoke-test-ml-with-security/build.gradle | 1 + 3 files changed, 17 insertions(+) diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/ml/action/PostCalendarEventsAction.java b/plugin/core/src/main/java/org/elasticsearch/xpack/ml/action/PostCalendarEventsAction.java index 0f897d2ed7c..2c97516b3f5 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/ml/action/PostCalendarEventsAction.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/ml/action/PostCalendarEventsAction.java @@ -82,6 +82,10 @@ public class PostCalendarEventsAction extends Action scheduledEvents) { this.calendarId = ExceptionsHelper.requireNonNull(calendarId, Calendar.ID.getPreferredName()); this.scheduledEvents = ExceptionsHelper.requireNonNull(scheduledEvents, EVENTS.getPreferredName()); + + if (scheduledEvents.isEmpty()) { + throw ExceptionsHelper.badRequestException("at least 1 event is required"); + } } public String getCalendarId() { diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml b/plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml index 5084b3574ee..9879103316d 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml @@ -584,3 +584,15 @@ - length: { events: 2 } - match: { events.0.description: ski } - match: { events.1.description: snow } + +--- +"Test post calendar events given empty events": + + - do: + catch: /at least 1 event is required/ + xpack.ml.post_calendar_events: + calendar_id: "foo" + body: > + { + "events": [] + } diff --git a/qa/smoke-test-ml-with-security/build.gradle b/qa/smoke-test-ml-with-security/build.gradle index 240a220706f..46708519a2f 100644 --- a/qa/smoke-test-ml-with-security/build.gradle +++ b/qa/smoke-test-ml-with-security/build.gradle @@ -22,6 +22,7 @@ integTestRunner { // know whether to expect an authorization exception or a validation exception 'ml/calendar_crud/Test cannot create calendar with name _all', 'ml/calendar_crud/Test PageParams with ID is invalid', + 'ml/calendar_crud/Test post calendar events given empty events', 'ml/custom_all_field/Test querying custom all field', 'ml/datafeeds_crud/Test delete datafeed with missing id', 'ml/datafeeds_crud/Test put datafeed referring to missing job_id', From ac2884f0344ba0212172833dc4bcc2e03d08e339 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Tue, 16 Jan 2018 10:11:12 +0100 Subject: [PATCH 12/15] Watcher: Fix NPE in watcher index template registry (elastic/x-pack-elasticsearch#3571) The current code throws an NPE, when there is no master node available in the cluster state. This commit adds an additional check for an existing master node. relates elastic/x-pack-elasticsearch#3569 Original commit: elastic/x-pack-elasticsearch@e0024b5170214f7dc05cd184db9056255887b6e6 --- .../watcher/support/WatcherIndexTemplateRegistry.java | 7 ++++++- .../support/WatcherIndexTemplateRegistryTests.java | 11 +++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java b/plugin/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java index abae21dfe65..465ce1a7e6d 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java @@ -81,10 +81,15 @@ public class WatcherIndexTemplateRegistry extends AbstractComponent implements C return; } + // no master node, exit immediately + DiscoveryNode masterNode = event.state().getNodes().getMasterNode(); + if (masterNode == null) { + return; + } + // if this node is newer than the master node, we probably need to add the history template, which might be newer than the // history template the master node has, so we need potentially add new templates despite being not the master node DiscoveryNode localNode = event.state().getNodes().getLocalNode(); - DiscoveryNode masterNode = event.state().getNodes().getMasterNode(); boolean localNodeVersionAfterMaster = localNode.getVersion().after(masterNode.getVersion()); if (event.localNodeMaster() || localNodeVersionAfterMaster) { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java index 8832aca8290..486a8b48d5e 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java @@ -128,6 +128,17 @@ public class WatcherIndexTemplateRegistryTests extends ESTestCase { verifyZeroInteractions(client); } + public void testThatMissingMasterNodeDoesNothing() { + DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build(); + + ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistry.TRIGGERED_TEMPLATE_NAME, + WatcherIndexTemplateRegistry.WATCHES_TEMPLATE_NAME, ".watch-history-6"), nodes); + registry.clusterChanged(event); + + verifyZeroInteractions(client); + } + private ClusterChangedEvent createClusterChangedEvent(List existingTemplateNames, DiscoveryNodes nodes) { ClusterChangedEvent event = mock(ClusterChangedEvent.class); when(event.localNodeMaster()).thenReturn(nodes.isLocalNodeElectedMaster()); From 2350b855074628ed42eb78c74bfdf4a068c4c5c2 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 16 Jan 2018 11:00:07 +0000 Subject: [PATCH 13/15] [ML] Improve error on get calendar when ml-meta index is missing (elastic/x-pack-elasticsearch#3576) Relates elastic/x-pack-elasticsearch#3016 Original commit: elastic/x-pack-elasticsearch@bc60a0cb72c83a2af2014a25692d1d6bb83be233 --- .../xpack/ml/job/persistence/JobProvider.java | 7 ++++++- .../test/resources/rest-api-spec/test/ml/calendar_crud.yml | 7 +++++++ qa/smoke-test-ml-with-security/build.gradle | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java b/plugin/core/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java index d9617038ef0..f811fcc1041 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java @@ -51,6 +51,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -1204,7 +1205,11 @@ public class JobProvider { @Override public void onFailure(Exception e) { - listener.onFailure(e); + if (e instanceof IndexNotFoundException) { + listener.onFailure(new ResourceNotFoundException("No calendar with id [" + calendarId + "]")); + } else { + listener.onFailure(e); + } } }, client::get); diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml b/plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml index 9879103316d..b604e779575 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml @@ -92,6 +92,13 @@ "job_ids": ["cal-job", "unknown-job"] } +--- +"Test get calendar given missing": + - do: + catch: /No calendar with id \[unknown\]/ + xpack.ml.get_calendars: + calendar_id: "unknown" + --- "Test PageParams": - do: diff --git a/qa/smoke-test-ml-with-security/build.gradle b/qa/smoke-test-ml-with-security/build.gradle index 46708519a2f..42f5dc0d5ab 100644 --- a/qa/smoke-test-ml-with-security/build.gradle +++ b/qa/smoke-test-ml-with-security/build.gradle @@ -20,6 +20,7 @@ integTestRunner { 'ml/ml_classic_analyze/Test analyze API with an analyzer that does what we used to do in native code', // Remove tests that are expected to throw an exception, because we cannot then // know whether to expect an authorization exception or a validation exception + 'ml/calendar_crud/Test get calendar given missing', 'ml/calendar_crud/Test cannot create calendar with name _all', 'ml/calendar_crud/Test PageParams with ID is invalid', 'ml/calendar_crud/Test post calendar events given empty events', From 0b8723015f223a5521fe2c68e163e8be1b629c98 Mon Sep 17 00:00:00 2001 From: Dimitrios Athanasiou Date: Tue, 16 Jan 2018 11:50:18 +0000 Subject: [PATCH 14/15] [ML] Capitalize first letter in error message Original commit: elastic/x-pack-elasticsearch@70628513579d759ee83ef7c85cd922c229e89066 --- .../elasticsearch/xpack/ml/action/PostCalendarEventsAction.java | 2 +- .../src/test/resources/rest-api-spec/test/ml/calendar_crud.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/ml/action/PostCalendarEventsAction.java b/plugin/core/src/main/java/org/elasticsearch/xpack/ml/action/PostCalendarEventsAction.java index 2c97516b3f5..eabec4a82e2 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/ml/action/PostCalendarEventsAction.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/ml/action/PostCalendarEventsAction.java @@ -84,7 +84,7 @@ public class PostCalendarEventsAction extends Action From 368c4fff569f6fc04eab924d2e2b6afb6ec9b203 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 16 Jan 2018 11:53:37 +0000 Subject: [PATCH 15/15] [ML] Expand job groups on scheduled events update (elastic/x-pack-elasticsearch#3577) Relates elastic/x-pack-elasticsearch#3016 Original commit: elastic/x-pack-elasticsearch@73334d8e01d29fca34d740f9c2cc202d51aeba57 --- .../xpack/ml/job/JobManager.java | 18 ++++++--- .../xpack/ml/job/JobManagerTests.java | 39 +++++++++++++++++++ 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index c85ef674dc7..bda2906159d 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -52,6 +52,7 @@ import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; import java.io.IOException; import java.util.ArrayList; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Set; @@ -132,6 +133,14 @@ public class JobManager extends AbstractComponent { return job; } + private Set expandJobIds(String expression, boolean allowNoJobs, ClusterState clusterState) { + MlMetadata mlMetadata = clusterState.getMetaData().custom(MLMetadataField.TYPE); + if (mlMetadata == null) { + mlMetadata = MlMetadata.EMPTY_METADATA; + } + return mlMetadata.expandJobIds(expression, allowNoJobs); + } + /** * Get the jobs that match the given {@code expression}. * Note that when the {@code jobId} is {@link MetaData#ALL} all jobs are returned. @@ -142,11 +151,8 @@ public class JobManager extends AbstractComponent { * @return A {@link QueryPage} containing the matching {@code Job}s */ public QueryPage expandJobs(String expression, boolean allowNoJobs, ClusterState clusterState) { + Set expandedJobIds = expandJobIds(expression, allowNoJobs, clusterState); MlMetadata mlMetadata = clusterState.getMetaData().custom(MLMetadataField.TYPE); - if (mlMetadata == null) { - mlMetadata = MlMetadata.EMPTY_METADATA; - } - Set expandedJobIds = mlMetadata.expandJobIds(expression, allowNoJobs); List jobs = new ArrayList<>(); for (String expandedJobId : expandedJobIds) { jobs.add(mlMetadata.getJobs().get(expandedJobId)); @@ -335,7 +341,9 @@ public class JobManager extends AbstractComponent { public void updateProcessOnCalendarChanged(List calendarJobIds) { ClusterState clusterState = clusterService.state(); - for (String jobId : calendarJobIds) { + Set expandedJobIds = new HashSet<>(); + calendarJobIds.stream().forEach(jobId -> expandedJobIds.addAll(expandJobIds(jobId, true, clusterState))); + for (String jobId : expandedJobIds) { if (isJobOpen(clusterState, jobId)) { updateJobProcessNotifier.submitJobUpdate(UpdateParams.scheduledEventsUpdate(jobId)); } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java index 860dea2b333..559fe9592d1 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java @@ -266,6 +266,45 @@ public class JobManagerTests extends ESTestCase { assertThat(capturedUpdateParams.get(1).isUpdateScheduledEvents(), is(true)); } + public void testUpdateProcessOnCalendarChanged_GivenGroups() { + Job.Builder job1 = buildJobBuilder("job-1"); + job1.setGroups(Collections.singletonList("group-1")); + Job.Builder job2 = buildJobBuilder("job-2"); + job2.setGroups(Collections.singletonList("group-1")); + Job.Builder job3 = buildJobBuilder("job-3"); + + MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); + mlMetadata.putJob(job1.build(), false); + mlMetadata.putJob(job2.build(), false); + mlMetadata.putJob(job3.build(), false); + + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); + addJobTask(job1.getId(), "node_id", JobState.OPENED, tasksBuilder); + addJobTask(job2.getId(), "node_id", JobState.OPENED, tasksBuilder); + addJobTask(job3.getId(), "node_id", JobState.OPENED, tasksBuilder); + + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder() + .putCustom(PersistentTasksCustomMetaData.TYPE, tasksBuilder.build()) + .putCustom(MLMetadataField.TYPE, mlMetadata.build())) + .build(); + when(clusterService.state()).thenReturn(clusterState); + + JobManager jobManager = createJobManager(); + + jobManager.updateProcessOnCalendarChanged(Collections.singletonList("group-1")); + + ArgumentCaptor updateParamsCaptor = ArgumentCaptor.forClass(UpdateParams.class); + verify(updateJobProcessNotifier, times(2)).submitJobUpdate(updateParamsCaptor.capture()); + + List capturedUpdateParams = updateParamsCaptor.getAllValues(); + assertThat(capturedUpdateParams.size(), equalTo(2)); + assertThat(capturedUpdateParams.get(0).getJobId(), equalTo(job1.getId())); + assertThat(capturedUpdateParams.get(0).isUpdateScheduledEvents(), is(true)); + assertThat(capturedUpdateParams.get(1).getJobId(), equalTo(job2.getId())); + assertThat(capturedUpdateParams.get(1).isUpdateScheduledEvents(), is(true)); + } + private Job.Builder createJob() { Detector.Builder d1 = new Detector.Builder("info_content", "domain"); d1.setOverFieldName("client");